{"cells":[{"cell_type":"code","source":["import pandas as pd\n","import numpy as np\n","import re\n","from tqdm.auto import tqdm\n","import seaborn as sns\n","import pickle\n","import time\n","import matplotlib\n","import matplotlib.pyplot as plt\n","from scipy.sparse import csr_matrix\n","import scipy.sparse as sps"],"metadata":{"id":"4JjxeQqQhNM9","executionInfo":{"status":"ok","timestamp":1765743611855,"user_tz":-60,"elapsed":3843,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":1,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"dVAa-v0bYujf"},"source":["# Load stuff previously defined"]},{"cell_type":"code","source":["import spacy\n","!pip install --quiet spacymoji\n","from spacymoji import Emoji\n","nlp = spacy.load('en_core_web_sm')\n","nlp.tokenizer.token_match = re.compile(\"^#\\w+$\").match\n","nlp.add_pipe(\"emoji\", first=True)\n","\n","import emoji\n","def get_emoji_regexp():\n"," # Sort emoji by length to make sure multi-character emojis are\n"," # matched first\n"," emojis = sorted(emoji.EMOJI_DATA, key=len, reverse=True)\n"," pattern = '(' + '|'.join(re.escape(u) for u in emojis) + ')'\n"," return re.compile(pattern)"],"metadata":{"id":"pwL9KcrOTLkd","colab":{"base_uri":"https://localhost:8080/"},"outputId":"575d6090-6d0d-433e-ab45-5546dbd4dda4","executionInfo":{"status":"ok","timestamp":1765743634911,"user_tz":-60,"elapsed":23048,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":2,"outputs":[{"output_type":"stream","name":"stderr","text":["<>:5: SyntaxWarning: invalid escape sequence '\\w'\n","<>:5: SyntaxWarning: invalid escape sequence '\\w'\n","/tmp/ipython-input-4268926626.py:5: SyntaxWarning: invalid escape sequence '\\w'\n"," nlp.tokenizer.token_match = re.compile(\"^#\\w+$\").match\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/608.4 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m604.2/608.4 kB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m608.4/608.4 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h"]}]},{"cell_type":"code","source":["!pip install --quiet igraph\n","import igraph as ig"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZGtTXttDlSXt","executionInfo":{"status":"ok","timestamp":1765744659249,"user_tz":-60,"elapsed":7193,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}},"outputId":"2d45daeb-2c7d-4236-b6ea-fdd55bf4c101"},"execution_count":28,"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/5.7 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[91m━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.5/5.7 MB\u001b[0m \u001b[31m13.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/5.7 MB\u001b[0m \u001b[31m45.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m60.2 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m41.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h"]}]},{"cell_type":"code","source":["class SemanticNetwork:\n"," def __init__(self,nlp):\n"," self.nlp=nlp\n","\n"," def _clean_text_fun(self, text, to_keep=True,\n"," kr_list = [\"ADJ\",\"ADV\",\"EMOJI\",\"HASH\",\"NOUN\",\"PROPN\",\"VERB\"]):\n","\n"," # apply spacy\n"," doc = self.nlp(\" \".join(get_emoji_regexp().split(text)))\n"," # collect spacy outomes in in a list\n"," out = list()\n"," for token in doc:\n"," out.append((token.text, token.text+token.whitespace_,\n"," token.lemma_.lower(), token.pos_,\n"," token.text.startswith(\"#\"),\n"," token.text.startswith(\"@\"),\n"," emoji.purely_emoji(token.text),\n"," (token.text.startswith(\"https:\") or token.text.startswith(\"http:\"))\n"," ))\n"," # turn it into a dataframe\n"," df2 = pd.DataFrame(out)\n"," df2.columns = ['text', 'full_text', 'lemma', 'PoS',\n"," 'is_hashtag', 'is_mention', 'is_emoji', 'is_link']\n"," # build a PoS column that identifies hashtags, mentions, etc\n"," df2.insert(loc = 4, column = 'myPoS', value = df2['PoS'])\n"," df2.loc[df2['is_hashtag']==True,'myPoS'] = 'HASH'\n"," df2.loc[df2['is_mention']==True,'myPoS'] = 'MENT'\n"," df2.loc[df2['is_emoji']==True,'myPoS'] = 'EMOJI'\n"," df2.loc[df2['is_link']==True,'myPoS'] = 'HTML'\n"," # replace emojis lemmas with their description\n"," for tmp in doc._.emoji:\n"," df2.loc[tmp[1],'lemma'] = tmp[0]+\" \"+tmp[2]\n"," # keep/remove only what is asked\n"," if to_keep:\n"," out = [j for (i,j) in zip(df2['myPoS'],df2['lemma']) if i in kr_list]\n"," else:\n"," out = [j for (i,j) in zip(df2['myPoS'],df2['lemma']) if i not in kr_list]\n"," return out\n","\n"," # with to_keep you can choose if you want to keep or remove certain PoS\n"," #\n"," # with kr_list you can choose which POS tags to keep or remove,\n"," # to be chosen from the following list\n"," #\n"," # Universal POS Tags http://universaldependencies.org/u/pos/\n"," #\n"," # \"ADJ\": \"adjective\",\n"," # \"ADP\": \"adposition\",\n"," # \"ADV\": \"adverb\",\n"," # \"AUX\": \"auxiliary\",\n"," # \"CONJ\": \"conjunction\",\n"," # \"CCONJ\": \"coordinating conjunction\",\n"," # \"DET\": \"determiner\",\n"," # \"INTJ\": \"interjection\",\n"," # \"NOUN\": \"noun\",\n"," # \"NUM\": \"numeral\",\n"," # \"PART\": \"particle\",\n"," # \"PRON\": \"pronoun\",\n"," # \"PROPN\": \"proper noun\",\n"," # \"PUNCT\": \"punctuation\",\n"," # \"SCONJ\": \"subordinating conjunction\",\n"," # \"SYM\": \"symbol\",\n"," # \"VERB\": \"verb\",\n"," # \"X\": \"other\",\n"," # \"EOL\": \"end of line\",\n"," # \"SPACE\": \"space\"\n"," #\n"," # Internal tags\n"," #\n"," # \"EMOJI\" emojis,\n"," # \"HASH\" hastags\n"," # \"HTML\" web links,\n"," # \"MENT\" mentions\n","\n"," def clean_text(self, df, to_keep=True,\n"," kr_list = [\"ADJ\",\"ADV\",\"EMOJI\",\"HASH\",\"NOUN\",\"PROPN\",\"VERB\"]):\n","\n"," self.df = df.copy()\n"," for i in tqdm(range(len(df))):\n"," text = df.loc[i,'translated']\n"," self.df.loc[[i],'clean list'] = pd.Series([\n"," self._clean_text_fun(text, to_keep, kr_list)\n"," ], index=[i])\n","\n"," # extracts occurrence matrix Nwd, also returns the\n"," # documents actually in use and the words dictionary\n"," #\n"," # words occurring less than n_min times are discarded\n"," # words occurring more than n_max times are discarded\n"," #\n"," # documents with zero active words are discarded\n","\n"," def get_Nwd(self, n_min=2, n_max=1e10):\n","\n"," # capture execution time\n"," tic = time.time()\n","\n"," # collection of (unique) words\n"," clean_texts_list = list(self.df['clean list'])\n"," words = np.unique([item for sublist in clean_texts_list \\\n"," for item in sublist])\n"," Nw = len(words) # number of words (so far)\n"," # documents list\n"," Nd = len(clean_texts_list) # number of documents (so far)\n"," documents = np.array(range(Nd))\n","\n"," # occurrence matrix for words in documents\n"," words_dict = dict(zip(words,range(Nw))) # words dictionary\n"," Nwd = csr_matrix((Nw, Nd), dtype = np.int8).toarray()\n"," for i in range(Nd):\n"," for j in clean_texts_list[i]:\n"," Nwd[words_dict[j],i] += 1\n","\n"," # identify words used less than n_min or more than n_max\n"," select = ((np.sum(Nwd,axis=1) >= n_min) & \\\n"," (np.sum(Nwd,axis=1) < n_max))\n"," # explicitly print the most frequent ones\n"," print('removing words...')\n"," with np.printoptions(threshold=np.inf):\n"," print(words[(np.sum(Nwd,axis=1)>=n_max)])\n"," # remove them\n"," Nwd = Nwd[select,:]\n"," words = words[select]\n"," # remove documents that do not contain words\n"," select = (np.sum(Nwd,axis=0)>0)\n"," Nwd = csr_matrix(Nwd[:,select])\n"," documents = documents[select]\n","\n"," # capture execution time\n"," print(f'Occurrence matrix: execution time {time.time()-tic} [s]')\n","\n"," self.Nwd = Nwd\n"," self.words = words\n"," self.documents = documents\n","\n"," # plot words occurrences\n"," plt.figure(figsize=(4, 3))\n"," plt.semilogy(-np.sort(-np.asarray(np.sum(Nwd,axis=1)).reshape(-1)))\n"," plt.grid(True)\n"," plt.xlabel('word id')\n"," plt.ylabel('# of occurrences')\n"," plt.title(\"words occurrencies\");\n"," plt.show()\n","\n"," # build other matrices\n"," # equally likely documents case!\n"," Pwd = Nwd/Nwd.sum(axis=0).flatten()/Nwd.shape[1]\n"," # words and document matrices\n"," pd = Pwd.sum(axis=0).flatten()\n"," Pww = (Pwd/pd).dot(Pwd.T)\n"," pw = Pwd.sum(axis=1).flatten()\n"," Pdd = (Pwd.T/pw).dot(Pwd)\n"," self.Pwd = Pwd\n"," self.Pdd = Pdd\n"," self.pd = pd"],"metadata":{"id":"93Roe9iggNDG","executionInfo":{"status":"ok","timestamp":1765743674077,"user_tz":-60,"elapsed":103,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":3,"outputs":[]},{"cell_type":"code","source":["def logg(x):\n"," y = np.log(x)\n"," y[x==0] = 0\n"," return y\n","\n","# NMI\n","def nmi_fn(A): # A = Pwc\n"," aw = A.sum(axis=1).flatten() # word probability\n"," ac = A.sum(axis=0).flatten() # class probability\n"," Hc = np.multiply(ac,-logg(ac)).sum() # class entropy\n"," A2 = ((A/ac).T/aw).T\n"," A2.data = logg(A2.data)\n"," y = (A.multiply(A2)).sum()/Hc\n"," return y\n","\n","# modularity\n","def modularity_fn(A):\n"," y = A.trace()-(A.sum(axis=0)*A.sum(axis=1)).item()\n"," return y\n","\n","# Ncut\n","def ncut_fn(A):\n"," y = ((A.sum(axis=0)-A.diagonal())/A.sum(axis=0)).mean()\n"," return y\n","\n","# Infomap - 1\n","def pagerank_fn(M,q,c=.85,it=60):\n"," r = q.copy() # ranking matrix, initialized to q (copy)\n"," for k in range(it): # slow cycle\n"," r = c*M.dot(r) + (1-c)*q\n"," return r\n","\n","# Infomap - 2\n","def _infomap_fn(v):\n"," y = -(v.data*logg(v.data/v.sum())).sum()\n"," return y\n","\n","# Infomap - 3\n","def infomap_rank_fn(Pdd):\n"," # transition matrix\n"," pd = Pdd.sum(axis=0).flatten()\n"," M = sps.csr_matrix(Pdd/pd)\n"," # pagerank vector - faster than r = pagerank_fn(M,q)\n"," G = ig.Graph.Adjacency((M > 0).toarray().tolist())\n"," G.es['weight'] = np.array(M[M.nonzero()])[0]\n"," r = G.pagerank(weights='weight')\n"," r = (sps.csr_matrix(np.array(r))).T\n"," return r\n","\n","# Infomap - 4\n","def infomap_fn(C,Pdd,r):\n"," pd = Pdd.sum(axis=0).flatten()\n"," M = Pdd/pd # transition matrix\n"," # extract vectors\n"," z = (C.T).dot(sps.diags(r.toarray().flatten()))\n"," q = sps.csr_matrix((1,z.shape[0]))\n"," c = .85\n"," for i in range(z.shape[0]):\n"," tmp = ((C[:,i].transpose()).dot(M)).dot(z[i].transpose())\n"," q[0,i] = (1-(1-c)*C[:,i].sum()/M.shape[0])*z[i].sum()-c*tmp[0,0]\n"," # extract statistics\n"," y = _infomap_fn(q)\n"," for i in range(z.shape[0]):\n"," y += _infomap_fn(sps.hstack([z[i],sps.csr_matrix([[q[0,i]]])]))\n"," # normalize\n"," y = (y/_infomap_fn(pd))-1\n"," return y"],"metadata":{"id":"r1QkkJSSngt4","executionInfo":{"status":"ok","timestamp":1765743698089,"user_tz":-60,"elapsed":45,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":4,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"ERt8qIRCKo6M"},"source":["# Load data and run BERTopic"]},{"cell_type":"code","source":["# load data\n","df = pd.read_excel(\"/content/tweets_greta_translated.xlsx\")\n","df.drop_duplicates(subset=[\"text\"],inplace=True)\n","df.reset_index(drop=True, inplace=True)"],"metadata":{"id":"5RDfMXIohuUv","executionInfo":{"status":"ok","timestamp":1765743756003,"user_tz":-60,"elapsed":1480,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["!pip install --quiet bertopic\n","from bertopic import BERTopic"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"THHR7ie0C_f1","outputId":"1889666f-4b1f-4615-a16b-a66bab237d52","executionInfo":{"status":"ok","timestamp":1765743820979,"user_tz":-60,"elapsed":51816,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":8,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/hdbscan/robust_single_linkage_.py:175: SyntaxWarning: invalid escape sequence '\\{'\n"," $max \\{ core_k(a), core_k(b), 1/\\alpha d(a,b) \\}$.\n"]}]},{"cell_type":"code","execution_count":31,"metadata":{"id":"6RRz8RjDIfCn","colab":{"base_uri":"https://localhost:8080/","height":1000},"outputId":"e056676e-aff7-480c-dce3-17dffaf1dee1","executionInfo":{"status":"ok","timestamp":1765744852453,"user_tz":-60,"elapsed":26884,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"outputs":[{"output_type":"execute_result","data":{"text/plain":[" Topic Count Name \\\n","0 -1 1330 -1_the_to_climatechange_of \n","1 0 684 0_climatechange_to_the_climate \n","2 1 314 1_sustainability_plastic_environment_recycling \n","3 2 167 2_sdgs_the_development_to \n","4 3 125 3_food_agriculture_farming_farmers \n","5 4 94 4_ice_arctic_sea_glaciers \n","6 5 80 5_africa_sdgs_in_nigeria \n","7 6 69 6_solar_energy_implement_panels \n","8 7 60 7_trees_forests_planting_forest \n","9 8 60 8_oil_carbon_emissions_spend \n","10 9 47 9_cdnpoli_canadians_canada_trudeau \n","11 10 46 10_electric_cars_norway_ev \n","12 11 43 11_water_worldwaterday_the_environment \n","13 12 41 12_sdgs_women_womensday_gender \n","14 13 40 13_power_renewables_wind_electricity \n","15 14 35 14_auspol_australia_australian_government \n","16 15 34 15_cyclone_mozambique_cycloneidai_idai \n","17 16 24 16_insects_insect_numbers_crashing \n","18 17 22 17_women_gender_internationalwomensday_to \n","19 18 21 18_ai_iot_robotics_automation \n","20 19 21 19_wildlife_species_environment_animals \n","21 20 21 20_temperature_record_warmest_records \n","22 21 19 21_airpollution_air_pollution_tb \n","23 22 19 22_mumbai_indian_india_28 \n","24 23 18 23_reef_coral_barrier_bleaching \n","25 24 17 24_eu_2050_vote_for \n","26 25 17 25_canada_warming_twice_report \n","27 26 17 26_deal_green_greennewdeal_new \n","28 27 17 27_methane_emissions_fossil_fuels \n","29 28 16 28_fish_lakes_fisheries_oceans \n","30 29 15 29_demswork4usa_progressives_healthcare_medicaid \n","31 30 15 30_attenborough_david_sir_documentary \n","32 31 15 31_cities_city_climate_httpstcotnuyoisan0 \n","33 32 13 32_secret_garden_tampa_climate \n","34 33 13 33_ocean_marine_oceans_whales \n","35 34 12 34_fracking_judge_drilling_wyoming \n","36 35 12 35_kashthefuturist_cberrl_amazingchevvolt_gezg... \n","37 36 12 36_nuclear_energy_httpstcowzwsvj71mq_nukes \n","38 37 11 37_health_immune_population_single \n","39 38 11 38_co2_atmosphere_were_was \n","\n"," Representation \\\n","0 [the, to, climatechange, of, and, is, in, on, ... \n","1 [climatechange, to, the, climate, and, in, is,... \n","2 [sustainability, plastic, environment, recycli... \n","3 [sdgs, the, development, to, on, goals, amp, i... \n","4 [food, agriculture, farming, farmers, to, soil... \n","5 [ice, arctic, sea, glaciers, melting, in, the,... \n","6 [africa, sdgs, in, nigeria, for, the, leavenoo... \n","7 [solar, energy, implement, panels, lets, solut... \n","8 [trees, forests, planting, forest, to, defores... \n","9 [oil, carbon, emissions, spend, and, its, lobb... \n","10 [cdnpoli, canadians, canada, trudeau, the, pri... \n","11 [electric, cars, norway, ev, crisis, air, impl... \n","12 [water, worldwaterday, the, environment, waste... \n","13 [sdgs, women, womensday, gender, iwd2019, equa... \n","14 [power, renewables, wind, electricity, renewab... \n","15 [auspol, australia, australian, government, ad... \n","16 [cyclone, mozambique, cycloneidai, idai, zimba... \n","17 [insects, insect, numbers, crashing, pesticide... \n","18 [women, gender, internationalwomensday, to, in... \n","19 [ai, iot, robotics, automation, smart, cities,... \n","20 [wildlife, species, environment, animals, tige... \n","21 [temperature, record, warmest, records, februa... \n","22 [airpollution, air, pollution, tb, asthma, smo... \n","23 [mumbai, indian, india, 28, globalwarming, nee... \n","24 [reef, coral, barrier, bleaching, reefs, great... \n","25 [eu, 2050, vote, for, european, climate, brexi... \n","26 [canada, warming, twice, report, rate, leaked,... \n","27 [deal, green, greennewdeal, new, hill, the, ca... \n","28 [methane, emissions, fossil, fuels, fracking, ... \n","29 [fish, lakes, fisheries, oceans, eutrophicatio... \n","30 [demswork4usa, progressives, healthcare, medic... \n","31 [attenborough, david, sir, documentary, bbc, p... \n","32 [cities, city, climate, httpstcotnuyoisan0, ve... \n","33 [secret, garden, tampa, climate, fighting, bac... \n","34 [ocean, marine, oceans, whales, lifeunderwater... \n","35 [fracking, judge, drilling, wyoming, lands, bl... \n","36 [kashthefuturist, cberrl, amazingchevvolt, gez... \n","37 [nuclear, energy, httpstcowzwsvj71mq, nukes, p... \n","38 [health, immune, population, single, threats, ... \n","39 [co2, atmosphere, were, was, 2016, usa, high, ... \n","\n"," Representative_Docs \n","0 [\"This is not about #climatechange anymore, th... \n","1 [It's elections time. Time for a new party: on... \n","2 [Did you, like me, think tins, glass and paper... \n","3 [The Sustainable\\nDevelopment Goals (#SDGs):\\n... \n","4 [Innovation is the central driving force which... \n","5 [ARCTIC SEA-ICE EXPLOSION : Largest Increase I... \n","6 [\"Africa bears the brunt of #ClimateChange and... \n","7 [Chinese scientists have invented #solar panel... \n","8 [Pakistan is planting 10 billion trees. To fig... \n","9 [According to a new report, the five largest s... \n","10 [#BigOil knew their products would cause #cli... \n","11 [Norway is banning cars from the centre of Osl... \n","12 [Just started a course this week on Water and ... \n","13 [I am very happy to share my experience with s... \n","14 [Wow. #Denmark is using the power of the ocean... \n","15 [This week Scott Morrison doubled down on Tony... \n","16 [At Least 150 Dead, 1.5 Million Impacted as Cy... \n","17 [Insect numbers are crashing in huge numbers d... \n","18 [@NamugerwaLeah @GretaThunberg @BBCAfrica @Ext... \n","19 [“Imagine creating a world that is driven by e... \n","20 [#planet #sustainable #sustainability \\n#susta... \n","21 [Animation showing the evolution of global mea... \n","22 [Air pollution is now more deadly than war, sm... \n","23 [Can someone urgently verify this before I fai... \n","24 [The Great Barrier Reef is being battered by #... \n","25 [Yesss! European youth is rising for the clima... \n","26 [Climate change is warming Canada at a rate tw... \n","27 [PRESS CONFERENCE: Tomorrow, I'll be unveiling... \n","28 [“The science is crystal clear, we need to pha... \n","29 [RT @SOCCOMProject: Oceans’ fever means fewer ... \n","30 [🔁#Florida CD26 #Election2020-Nov-3.\\n\\nSuppor... \n","31 [Sir David Attenborough to present climate cha... \n","32 [Venice\\n\\nSo full of history, art, and poetry... \n","33 [Northwestern tribes and the University of Was... \n","34 [30×30: groundbreaking scientific study maps o... \n","35 [It has now become clear, with guidance from t... \n","36 [@Jackthelad1947 @NeurozoInnovat1 @ristori20 @... \n","37 [Given that a new nuclear power plant getting ... \n","38 [#Climatechange has been identified as the big... \n","39 [@realDonaldTrump @foxandfriends Earth's abili... "],"text/html":["\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
TopicCountNameRepresentationRepresentative_Docs
0-11330-1_the_to_climatechange_of[the, to, climatechange, of, and, is, in, on, ...[\"This is not about #climatechange anymore, th...
106840_climatechange_to_the_climate[climatechange, to, the, climate, and, in, is,...[It's elections time. Time for a new party: on...
213141_sustainability_plastic_environment_recycling[sustainability, plastic, environment, recycli...[Did you, like me, think tins, glass and paper...
321672_sdgs_the_development_to[sdgs, the, development, to, on, goals, amp, i...[The Sustainable\\nDevelopment Goals (#SDGs):\\n...
431253_food_agriculture_farming_farmers[food, agriculture, farming, farmers, to, soil...[Innovation is the central driving force which...
54944_ice_arctic_sea_glaciers[ice, arctic, sea, glaciers, melting, in, the,...[ARCTIC SEA-ICE EXPLOSION : Largest Increase I...
65805_africa_sdgs_in_nigeria[africa, sdgs, in, nigeria, for, the, leavenoo...[\"Africa bears the brunt of #ClimateChange and...
76696_solar_energy_implement_panels[solar, energy, implement, panels, lets, solut...[Chinese scientists have invented #solar panel...
87607_trees_forests_planting_forest[trees, forests, planting, forest, to, defores...[Pakistan is planting 10 billion trees. To fig...
98608_oil_carbon_emissions_spend[oil, carbon, emissions, spend, and, its, lobb...[According to a new report, the five largest s...
109479_cdnpoli_canadians_canada_trudeau[cdnpoli, canadians, canada, trudeau, the, pri...[#BigOil knew their products would cause #cli...
11104610_electric_cars_norway_ev[electric, cars, norway, ev, crisis, air, impl...[Norway is banning cars from the centre of Osl...
12114311_water_worldwaterday_the_environment[water, worldwaterday, the, environment, waste...[Just started a course this week on Water and ...
13124112_sdgs_women_womensday_gender[sdgs, women, womensday, gender, iwd2019, equa...[I am very happy to share my experience with s...
14134013_power_renewables_wind_electricity[power, renewables, wind, electricity, renewab...[Wow. #Denmark is using the power of the ocean...
15143514_auspol_australia_australian_government[auspol, australia, australian, government, ad...[This week Scott Morrison doubled down on Tony...
16153415_cyclone_mozambique_cycloneidai_idai[cyclone, mozambique, cycloneidai, idai, zimba...[At Least 150 Dead, 1.5 Million Impacted as Cy...
17162416_insects_insect_numbers_crashing[insects, insect, numbers, crashing, pesticide...[Insect numbers are crashing in huge numbers d...
18172217_women_gender_internationalwomensday_to[women, gender, internationalwomensday, to, in...[@NamugerwaLeah @GretaThunberg @BBCAfrica @Ext...
19182118_ai_iot_robotics_automation[ai, iot, robotics, automation, smart, cities,...[“Imagine creating a world that is driven by e...
20192119_wildlife_species_environment_animals[wildlife, species, environment, animals, tige...[#planet #sustainable #sustainability \\n#susta...
21202120_temperature_record_warmest_records[temperature, record, warmest, records, februa...[Animation showing the evolution of global mea...
22211921_airpollution_air_pollution_tb[airpollution, air, pollution, tb, asthma, smo...[Air pollution is now more deadly than war, sm...
23221922_mumbai_indian_india_28[mumbai, indian, india, 28, globalwarming, nee...[Can someone urgently verify this before I fai...
24231823_reef_coral_barrier_bleaching[reef, coral, barrier, bleaching, reefs, great...[The Great Barrier Reef is being battered by #...
25241724_eu_2050_vote_for[eu, 2050, vote, for, european, climate, brexi...[Yesss! European youth is rising for the clima...
26251725_canada_warming_twice_report[canada, warming, twice, report, rate, leaked,...[Climate change is warming Canada at a rate tw...
27261726_deal_green_greennewdeal_new[deal, green, greennewdeal, new, hill, the, ca...[PRESS CONFERENCE: Tomorrow, I'll be unveiling...
28271727_methane_emissions_fossil_fuels[methane, emissions, fossil, fuels, fracking, ...[“The science is crystal clear, we need to pha...
29281628_fish_lakes_fisheries_oceans[fish, lakes, fisheries, oceans, eutrophicatio...[RT @SOCCOMProject: Oceans’ fever means fewer ...
30291529_demswork4usa_progressives_healthcare_medicaid[demswork4usa, progressives, healthcare, medic...[🔁#Florida CD26 #Election2020-Nov-3.\\n\\nSuppor...
31301530_attenborough_david_sir_documentary[attenborough, david, sir, documentary, bbc, p...[Sir David Attenborough to present climate cha...
32311531_cities_city_climate_httpstcotnuyoisan0[cities, city, climate, httpstcotnuyoisan0, ve...[Venice\\n\\nSo full of history, art, and poetry...
33321332_secret_garden_tampa_climate[secret, garden, tampa, climate, fighting, bac...[Northwestern tribes and the University of Was...
34331333_ocean_marine_oceans_whales[ocean, marine, oceans, whales, lifeunderwater...[30×30: groundbreaking scientific study maps o...
35341234_fracking_judge_drilling_wyoming[fracking, judge, drilling, wyoming, lands, bl...[It has now become clear, with guidance from t...
36351235_kashthefuturist_cberrl_amazingchevvolt_gezg...[kashthefuturist, cberrl, amazingchevvolt, gez...[@Jackthelad1947 @NeurozoInnovat1 @ristori20 @...
37361236_nuclear_energy_httpstcowzwsvj71mq_nukes[nuclear, energy, httpstcowzwsvj71mq, nukes, p...[Given that a new nuclear power plant getting ...
38371137_health_immune_population_single[health, immune, population, single, threats, ...[#Climatechange has been identified as the big...
39381138_co2_atmosphere_were_was[co2, atmosphere, were, was, 2016, usa, high, ...[@realDonaldTrump @foxandfriends Earth's abili...
\n","
\n","
\n","\n","
\n"," \n","\n"," \n","\n"," \n","
\n","\n","\n","
\n"," \n","\n","\n","\n"," \n","
\n","\n","
\n","
\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","summary":"{\n \"name\": \"topic_model\",\n \"rows\": 40,\n \"fields\": [\n {\n \"column\": \"Topic\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 11,\n \"min\": -1,\n \"max\": 38,\n \"num_unique_values\": 40,\n \"samples\": [\n 18,\n 15,\n 14\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Count\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 231,\n \"min\": 11,\n \"max\": 1330,\n \"num_unique_values\": 27,\n \"samples\": [\n 60,\n 40,\n 47\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Name\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 40,\n \"samples\": [\n \"18_ai_iot_robotics_automation\",\n \"15_cyclone_mozambique_cycloneidai_idai\",\n \"14_auspol_australia_australian_government\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Representation\",\n \"properties\": {\n \"dtype\": \"object\",\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Representative_Docs\",\n \"properties\": {\n \"dtype\": \"object\",\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"}},"metadata":{},"execution_count":31}],"source":["# run BERTopic ... it takes about 1 min\n","topic_model = BERTopic(nr_topics=\"auto\")\n","docs = df['translated']\n","topics, probs = topic_model.fit_transform(docs)\n","topic_model.get_topic_info()"]},{"cell_type":"code","source":["# reduce outliers (they are very many)\n","new_topics = topic_model.reduce_outliers(docs, topics)\n","topic_model.update_topics(docs, topics=new_topics)\n","topic_model.get_topic_info()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"Uy1UEVPpize0","executionInfo":{"status":"ok","timestamp":1765744852866,"user_tz":-60,"elapsed":401,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}},"outputId":"f6cd5b76-e4f3-40e1-9199-1ec6fed81e17"},"execution_count":32,"outputs":[{"output_type":"stream","name":"stderr","text":["2025-12-14 20:40:52,645 - BERTopic - WARNING: Using a custom list of topic assignments may lead to errors if topic reduction techniques are used afterwards. Make sure that manually assigning topics is the last step in the pipeline.Note that topic embeddings will also be created through weightedc-TF-IDF embeddings instead of centroid embeddings.\n"]},{"output_type":"execute_result","data":{"text/plain":[" Topic Count Name \\\n","0 0 1343 0_climatechange_the_to_climate \n","1 1 479 1_sustainability_environment_plastic_and \n","2 2 273 2_sdgs_the_development_to \n","3 3 143 3_food_agriculture_farming_farmers \n","4 4 104 4_ice_arctic_sea_melting \n","5 5 92 5_africa_sdgs_in_for \n","6 6 96 6_solar_energy_implement_panelsnotpipelines \n","7 7 69 7_trees_forests_planting_forest \n","8 8 85 8_oil_carbon_emissions_gas \n","9 9 66 9_cdnpoli_canada_canadians_price \n","10 10 55 10_electric_cars_norway_tesla \n","11 11 63 11_water_environment_the_of \n","12 12 46 12_sdgs_women_gender_womensday \n","13 13 69 13_renewables_power_renewableenergy_wind \n","14 14 53 14_auspol_australia_australian_ausvotes2019 \n","15 15 35 15_cyclone_mozambique_cycloneidai_idai \n","16 16 26 16_insects_insect_numbers_pesticides \n","17 17 27 17_women_girls_gender_that \n","18 18 31 18_ai_iot_robotics_ronaldvanloon \n","19 19 32 19_wildlife_nature_species_animals \n","20 20 37 20_temperature_phd_kidsbeachgarden_masters \n","21 21 38 21_air_pollution_airpollution_smoking \n","22 22 20 22_mumbai_indian_india_28 \n","23 23 19 23_reef_coral_barrier_bleaching \n","24 24 24 24_eu_for_european_2050 \n","25 25 34 25_warming_canada_global_twice \n","26 26 35 26_green_deal_greennewdeal_new \n","27 27 26 27_methane_emissions_fuels_than \n","28 28 17 28_fish_lakes_fisheries_oceans \n","29 29 18 29_demswork4usa_progressives_healthcare_medicaid \n","30 30 20 30_attenborough_david_sir_documentary \n","31 31 28 31_cities_city_climate_change \n","32 32 21 32_garden_climate_back_secret \n","33 33 27 33_ocean_oceans_marine_amp \n","34 34 15 34_fracking_judge_drilling_court \n","35 35 14 35_kashthefuturist_jackthelad1947_charluv2011_... \n","36 36 18 36_nuclear_energy_policy_greenpeace \n","37 37 23 37_health_problem_is_immune \n","38 38 26 38_co2_were_atmosphere_million \n","\n"," Representation \\\n","0 [climatechange, the, to, climate, of, is, and,... \n","1 [sustainability, environment, plastic, and, fo... \n","2 [sdgs, the, development, to, amp, on, for, of,... \n","3 [food, agriculture, farming, farmers, soil, to... \n","4 [ice, arctic, sea, melting, glaciers, in, the,... \n","5 [africa, sdgs, in, for, nigeria, of, the, amp,... \n","6 [solar, energy, implement, panelsnotpipelines,... \n","7 [trees, forests, planting, forest, deforestati... \n","8 [oil, carbon, emissions, gas, and, its, climat... \n","9 [cdnpoli, canada, canadians, price, trudeau, t... \n","10 [electric, cars, norway, tesla, ev, energy, cr... \n","11 [water, environment, the, of, amp, is, and, wo... \n","12 [sdgs, women, gender, womensday, iwd2019, girl... \n","13 [renewables, power, renewableenergy, wind, ene... \n","14 [auspol, australia, australian, ausvotes2019, ... \n","15 [cyclone, mozambique, cycloneidai, idai, zimba... \n","16 [insects, insect, numbers, pesticides, crashin... \n","17 [women, girls, gender, that, to, international... \n","18 [ai, iot, robotics, ronaldvanloon, automation,... \n","19 [wildlife, nature, species, animals, environme... \n","20 [temperature, phd, kidsbeachgarden, masters, b... \n","21 [air, pollution, airpollution, smoking, health... \n","22 [mumbai, indian, india, 28, globalwarming, nee... \n","23 [reef, coral, barrier, bleaching, reefs, great... \n","24 [eu, for, european, 2050, vote, remain, climat... \n","25 [warming, canada, global, twice, report, rate,... \n","26 [green, deal, greennewdeal, new, capitalism, v... \n","27 [methane, emissions, fuels, than, fossil, burn... \n","28 [fish, lakes, fisheries, oceans, study, eutrop... \n","29 [demswork4usa, progressives, healthcare, medic... \n","30 [attenborough, david, sir, documentary, presen... \n","31 [cities, city, climate, change, of, how, it, w... \n","32 [garden, climate, back, secret, protect, vulne... \n","33 [ocean, oceans, marine, amp, whales, the, prot... \n","34 [fracking, judge, drilling, court, mining, gas... \n","35 [kashthefuturist, jackthelad1947, charluv2011,... \n","36 [nuclear, energy, policy, greenpeace, uranium,... \n","37 [health, problem, is, immune, population, sing... \n","38 [co2, were, atmosphere, million, levels, there... \n","\n"," Representative_Docs \n","0 [It's elections time. Time for a new party: on... \n","1 [Did you, like me, think tins, glass and paper... \n","2 [The Sustainable\\nDevelopment Goals (#SDGs):\\n... \n","3 [Innovation is the central driving force which... \n","4 [ARCTIC SEA-ICE EXPLOSION : Largest Increase I... \n","5 [\"Africa bears the brunt of #ClimateChange and... \n","6 [Chinese scientists have invented #solar panel... \n","7 [Pakistan is planting 10 billion trees. To fig... \n","8 [According to a new report, the five largest s... \n","9 [#BigOil knew their products would cause #cli... \n","10 [Norway is banning cars from the centre of Osl... \n","11 [Just started a course this week on Water and ... \n","12 [I am very happy to share my experience with s... \n","13 [Wow. #Denmark is using the power of the ocean... \n","14 [This week Scott Morrison doubled down on Tony... \n","15 [At Least 150 Dead, 1.5 Million Impacted as Cy... \n","16 [Insect numbers are crashing in huge numbers d... \n","17 [@NamugerwaLeah @GretaThunberg @BBCAfrica @Ext... \n","18 [“Imagine creating a world that is driven by e... \n","19 [#planet #sustainable #sustainability \\n#susta... \n","20 [Animation showing the evolution of global mea... \n","21 [Air pollution is now more deadly than war, sm... \n","22 [Can someone urgently verify this before I fai... \n","23 [The Great Barrier Reef is being battered by #... \n","24 [Yesss! European youth is rising for the clima... \n","25 [Climate change is warming Canada at a rate tw... \n","26 [PRESS CONFERENCE: Tomorrow, I'll be unveiling... \n","27 [“The science is crystal clear, we need to pha... \n","28 [RT @SOCCOMProject: Oceans’ fever means fewer ... \n","29 [🔁#Florida CD26 #Election2020-Nov-3.\\n\\nSuppor... \n","30 [Sir David Attenborough to present climate cha... \n","31 [Venice\\n\\nSo full of history, art, and poetry... \n","32 [Northwestern tribes and the University of Was... \n","33 [30×30: groundbreaking scientific study maps o... \n","34 [It has now become clear, with guidance from t... \n","35 [@Jackthelad1947 @NeurozoInnovat1 @ristori20 @... \n","36 [Given that a new nuclear power plant getting ... \n","37 [#Climatechange has been identified as the big... \n","38 [@realDonaldTrump @foxandfriends Earth's abili... "],"text/html":["\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
TopicCountNameRepresentationRepresentative_Docs
0013430_climatechange_the_to_climate[climatechange, the, to, climate, of, is, and,...[It's elections time. Time for a new party: on...
114791_sustainability_environment_plastic_and[sustainability, environment, plastic, and, fo...[Did you, like me, think tins, glass and paper...
222732_sdgs_the_development_to[sdgs, the, development, to, amp, on, for, of,...[The Sustainable\\nDevelopment Goals (#SDGs):\\n...
331433_food_agriculture_farming_farmers[food, agriculture, farming, farmers, soil, to...[Innovation is the central driving force which...
441044_ice_arctic_sea_melting[ice, arctic, sea, melting, glaciers, in, the,...[ARCTIC SEA-ICE EXPLOSION : Largest Increase I...
55925_africa_sdgs_in_for[africa, sdgs, in, for, nigeria, of, the, amp,...[\"Africa bears the brunt of #ClimateChange and...
66966_solar_energy_implement_panelsnotpipelines[solar, energy, implement, panelsnotpipelines,...[Chinese scientists have invented #solar panel...
77697_trees_forests_planting_forest[trees, forests, planting, forest, deforestati...[Pakistan is planting 10 billion trees. To fig...
88858_oil_carbon_emissions_gas[oil, carbon, emissions, gas, and, its, climat...[According to a new report, the five largest s...
99669_cdnpoli_canada_canadians_price[cdnpoli, canada, canadians, price, trudeau, t...[#BigOil knew their products would cause #cli...
10105510_electric_cars_norway_tesla[electric, cars, norway, tesla, ev, energy, cr...[Norway is banning cars from the centre of Osl...
11116311_water_environment_the_of[water, environment, the, of, amp, is, and, wo...[Just started a course this week on Water and ...
12124612_sdgs_women_gender_womensday[sdgs, women, gender, womensday, iwd2019, girl...[I am very happy to share my experience with s...
13136913_renewables_power_renewableenergy_wind[renewables, power, renewableenergy, wind, ene...[Wow. #Denmark is using the power of the ocean...
14145314_auspol_australia_australian_ausvotes2019[auspol, australia, australian, ausvotes2019, ...[This week Scott Morrison doubled down on Tony...
15153515_cyclone_mozambique_cycloneidai_idai[cyclone, mozambique, cycloneidai, idai, zimba...[At Least 150 Dead, 1.5 Million Impacted as Cy...
16162616_insects_insect_numbers_pesticides[insects, insect, numbers, pesticides, crashin...[Insect numbers are crashing in huge numbers d...
17172717_women_girls_gender_that[women, girls, gender, that, to, international...[@NamugerwaLeah @GretaThunberg @BBCAfrica @Ext...
18183118_ai_iot_robotics_ronaldvanloon[ai, iot, robotics, ronaldvanloon, automation,...[“Imagine creating a world that is driven by e...
19193219_wildlife_nature_species_animals[wildlife, nature, species, animals, environme...[#planet #sustainable #sustainability \\n#susta...
20203720_temperature_phd_kidsbeachgarden_masters[temperature, phd, kidsbeachgarden, masters, b...[Animation showing the evolution of global mea...
21213821_air_pollution_airpollution_smoking[air, pollution, airpollution, smoking, health...[Air pollution is now more deadly than war, sm...
22222022_mumbai_indian_india_28[mumbai, indian, india, 28, globalwarming, nee...[Can someone urgently verify this before I fai...
23231923_reef_coral_barrier_bleaching[reef, coral, barrier, bleaching, reefs, great...[The Great Barrier Reef is being battered by #...
24242424_eu_for_european_2050[eu, for, european, 2050, vote, remain, climat...[Yesss! European youth is rising for the clima...
25253425_warming_canada_global_twice[warming, canada, global, twice, report, rate,...[Climate change is warming Canada at a rate tw...
26263526_green_deal_greennewdeal_new[green, deal, greennewdeal, new, capitalism, v...[PRESS CONFERENCE: Tomorrow, I'll be unveiling...
27272627_methane_emissions_fuels_than[methane, emissions, fuels, than, fossil, burn...[“The science is crystal clear, we need to pha...
28281728_fish_lakes_fisheries_oceans[fish, lakes, fisheries, oceans, study, eutrop...[RT @SOCCOMProject: Oceans’ fever means fewer ...
29291829_demswork4usa_progressives_healthcare_medicaid[demswork4usa, progressives, healthcare, medic...[🔁#Florida CD26 #Election2020-Nov-3.\\n\\nSuppor...
30302030_attenborough_david_sir_documentary[attenborough, david, sir, documentary, presen...[Sir David Attenborough to present climate cha...
31312831_cities_city_climate_change[cities, city, climate, change, of, how, it, w...[Venice\\n\\nSo full of history, art, and poetry...
32322132_garden_climate_back_secret[garden, climate, back, secret, protect, vulne...[Northwestern tribes and the University of Was...
33332733_ocean_oceans_marine_amp[ocean, oceans, marine, amp, whales, the, prot...[30×30: groundbreaking scientific study maps o...
34341534_fracking_judge_drilling_court[fracking, judge, drilling, court, mining, gas...[It has now become clear, with guidance from t...
35351435_kashthefuturist_jackthelad1947_charluv2011_...[kashthefuturist, jackthelad1947, charluv2011,...[@Jackthelad1947 @NeurozoInnovat1 @ristori20 @...
36361836_nuclear_energy_policy_greenpeace[nuclear, energy, policy, greenpeace, uranium,...[Given that a new nuclear power plant getting ...
37372337_health_problem_is_immune[health, problem, is, immune, population, sing...[#Climatechange has been identified as the big...
38382638_co2_were_atmosphere_million[co2, were, atmosphere, million, levels, there...[@realDonaldTrump @foxandfriends Earth's abili...
\n","
\n","
\n","\n","
\n"," \n","\n"," \n","\n"," \n","
\n","\n","\n","
\n"," \n","\n","\n","\n"," \n","
\n","\n","
\n","
\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","summary":"{\n \"name\": \"topic_model\",\n \"rows\": 39,\n \"fields\": [\n {\n \"column\": \"Topic\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 11,\n \"min\": 0,\n \"max\": 38,\n \"num_unique_values\": 39,\n \"samples\": [\n 33,\n 36,\n 4\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Count\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 221,\n \"min\": 14,\n \"max\": 1343,\n \"num_unique_values\": 32,\n \"samples\": [\n 15,\n 26,\n 34\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Name\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 39,\n \"samples\": [\n \"33_ocean_oceans_marine_amp\",\n \"36_nuclear_energy_policy_greenpeace\",\n \"4_ice_arctic_sea_melting\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Representation\",\n \"properties\": {\n \"dtype\": \"object\",\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Representative_Docs\",\n \"properties\": {\n \"dtype\": \"object\",\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"}},"metadata":{},"execution_count":32}]},{"cell_type":"code","source":["topic_model.visualize_topics()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":667},"id":"0NJP2djOez5S","outputId":"1886fc71-e463-4da3-8bca-d0e7cf70cab3","executionInfo":{"status":"ok","timestamp":1765744852921,"user_tz":-60,"elapsed":54,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":33,"outputs":[{"output_type":"display_data","data":{"text/html":["\n","\n","\n","
\n","
\n","\n",""]},"metadata":{}}]},{"cell_type":"code","source":["topic_model.visualize_heatmap()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":837},"id":"JLqCLerTe5W1","outputId":"47610ed3-f0c8-4ff4-8746-032c43defc34","executionInfo":{"status":"ok","timestamp":1765744852985,"user_tz":-60,"elapsed":60,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":34,"outputs":[{"output_type":"display_data","data":{"text/html":["\n","\n","\n","
\n","
\n","\n",""]},"metadata":{}}]},{"cell_type":"code","source":["topic_model.visualize_documents(docs)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":787},"id":"Am6qJRHMfbLz","outputId":"81162800-2a3d-4a50-c88a-0f481dfb6bdb","executionInfo":{"status":"ok","timestamp":1765744875092,"user_tz":-60,"elapsed":22106,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":35,"outputs":[{"output_type":"display_data","data":{"text/html":["\n","\n","\n","
\n","
\n","\n",""]},"metadata":{}}]},{"cell_type":"code","source":["topic_model.visualize_barchart(top_n_topics=100, n_words=15)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"1aPhNrsBfqKs","outputId":"5b784fc2-3426-492b-b847-4cca3a3bb618","executionInfo":{"status":"ok","timestamp":1765744875104,"user_tz":-60,"elapsed":3,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":36,"outputs":[{"output_type":"display_data","data":{"text/html":["\n","\n","\n","
\n","
\n","\n",""]},"metadata":{}}]},{"cell_type":"markdown","source":["# Get measures on topic assignment"],"metadata":{"id":"6AGeC7r3mg21"}},{"cell_type":"code","source":["# load stored data\n","with open(\"/content/semantic_net.pkl\", 'rb') as f:\n"," in_data = pickle.load(f)"],"metadata":{"id":"rXMa7cU8j1Ld","executionInfo":{"status":"ok","timestamp":1765744346669,"user_tz":-60,"elapsed":525,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":18,"outputs":[]},{"cell_type":"code","source":["# build C matrix\n","b_topics = np.array(new_topics)\n","b_topics = b_topics[in_data.documents]\n","C = sps.csr_matrix((len(b_topics),b_topics.max()+1))\n","for i in range(len(b_topics)):\n"," C[i,b_topics[i]] = 1\n","\n","# builds topic matrices\n","Pwc = in_data.Pwd.dot(C) # joint word + class probability\n","Pcc = ((C.T).dot(in_data.Pdd)).dot(C) # joint class + class probability\n","pc = Pcc.sum(axis=0)\n","\n","# show number of topics, and size\n","plt.bar(np.array(range(C.shape[1])),np.array(C.sum(axis=0))[0])\n","plt.xlabel(\"topic #\")\n","plt.ylabel(\"# of documents\");"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":540},"id":"hsaGdrRikKny","executionInfo":{"status":"ok","timestamp":1765745026841,"user_tz":-60,"elapsed":1484,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}},"outputId":"dc561891-4d40-4d5a-8ea5-17ef02e018fa"},"execution_count":40,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/scipy/sparse/_index.py:168: SparseEfficiencyWarning:\n","\n","Changing the sparsity structure of a csr_matrix is expensive. lil and dok are more efficient.\n","\n"]},{"output_type":"display_data","data":{"text/plain":["
"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAkkAAAGwCAYAAAC99fF4AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAANKdJREFUeJzt3X9YVHXe//HXIAKKAv4IkEQkNZX8mSZNplmyYnKXre5dbrRZcclaUKlZ6r1JZe1itpnimm5bm3WvZj+11cokTdxNRERJUyJrSTEdaCMY8QcinO8ffp27kdPK4MAM+Hxc17mumc/5zDnvj5/t4rWfc+aMxTAMQwAAAHDi4+kCAAAAvBEhCQAAwAQhCQAAwAQhCQAAwAQhCQAAwAQhCQAAwAQhCQAAwISvpwtoDmpra3XkyBG1b99eFovF0+UAAIB6MAxDx44dU0REhHx8XF8XIiTVw5EjRxQZGenpMgAAQAMUFxera9euLn+OkFQP7du3l3T2HzkoKMjD1QAAgPqw2+2KjIx0/B13FSGpHs5dYgsKCiIkAQDQzDT0Vhlu3AYAADBBSAIAADBBSAIAADBBSAIAADBBSAIAADBBSAIAADBBSAIAADBBSAIAADBBSAIAADBBSAIAADBBSAIAADBBSAIAADBBSAIAADBBSAIAADBBSAIAADDh6+kCAABAy9Z99gcX7PPt/IQmqMQ1rCQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACYICQBAACY8GhI2rp1q2655RZFRETIYrFo7dq1jn3V1dWaNWuW+vfvr8DAQEVEROjuu+/WkSNHnI5RVlamxMREBQUFKSQkRElJSaqsrHTqs2fPHo0YMUIBAQGKjIzUggULmmJ4AACgGfNoSDp+/LgGDhyopUuX1tl34sQJ7dq1S3PnztWuXbv03nvvqbCwULfeeqtTv8TERO3bt0+ZmZlav369tm7dquTkZMd+u92uMWPGKCoqSnl5eXruuef05JNP6qWXXmr08QEAgObLYhiG4ekiJMlisWjNmjW67bbbfrZPbm6uhg0bpoMHD6pbt24qKChQTEyMcnNzNXToUEnShg0bNG7cOB0+fFgRERFatmyZfve738lms8nPz0+SNHv2bK1du1ZffvllvWqz2+0KDg5WRUWFgoKCLnqsAABcSrrP/uCCfb6dn+D2817s3+9mdU9SRUWFLBaLQkJCJEnZ2dkKCQlxBCRJiouLk4+Pj3Jychx9Ro4c6QhIkhQfH6/CwkL9+OOPpuepqqqS3W532gAAwKWl2YSkU6dOadasWfr1r3/tSIM2m02hoaFO/Xx9fdWxY0fZbDZHn7CwMKc+596f63O+9PR0BQcHO7bIyEh3DwcAAHi5ZhGSqqurdfvtt8swDC1btqzRzzdnzhxVVFQ4tuLi4kY/JwAA8C6+ni7gQs4FpIMHD2rz5s1O1xTDw8NVWlrq1P/MmTMqKytTeHi4o09JSYlTn3Pvz/U5n7+/v/z9/d05DAAA0Mx49UrSuYB04MABffLJJ+rUqZPTfqvVqvLycuXl5TnaNm/erNraWsXGxjr6bN26VdXV1Y4+mZmZ6t27tzp06NA0AwEAAM2OR0NSZWWl8vPzlZ+fL0kqKipSfn6+Dh06pOrqav3qV7/Szp07tXLlStXU1Mhms8lms+n06dOSpL59+2rs2LGaMmWKduzYoc8++0ypqamaNGmSIiIiJEl33nmn/Pz8lJSUpH379unNN9/U4sWLNWPGDE8NGwAANAMefQTAli1bdOONN9Zpnzx5sp588klFR0ebfu7TTz/VqFGjJJ19mGRqaqrWrVsnHx8fTZw4URkZGWrXrp2j/549e5SSkqLc3Fx17txZDz74oGbNmlXvOnkEAAAADddcHwHgNc9J8maEJAAAGq65hiSvvicJAADAUwhJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJghJAAAAJjwakrZu3apbbrlFERERslgsWrt2rdN+wzCUlpamLl26qE2bNoqLi9OBAwec+pSVlSkxMVFBQUEKCQlRUlKSKisrnfrs2bNHI0aMUEBAgCIjI7VgwYLGHhoAAGjmPBqSjh8/roEDB2rp0qWm+xcsWKCMjAwtX75cOTk5CgwMVHx8vE6dOuXok5iYqH379ikzM1Pr16/X1q1blZyc7Nhvt9s1ZswYRUVFKS8vT88995yefPJJvfTSS40+PgAA0HxZDMMwPF2EJFksFq1Zs0a33XabpLOrSBEREXrkkUc0c+ZMSVJFRYXCwsK0YsUKTZo0SQUFBYqJiVFubq6GDh0qSdqwYYPGjRunw4cPKyIiQsuWLdPvfvc72Ww2+fn5SZJmz56ttWvX6ssvv6xXbXa7XcHBwaqoqFBQUJD7Bw8AQAvWffYHF+zz7fwEt5/3Yv9+e+09SUVFRbLZbIqLi3O0BQcHKzY2VtnZ2ZKk7OxshYSEOAKSJMXFxcnHx0c5OTmOPiNHjnQEJEmKj49XYWGhfvzxR9NzV1VVyW63O20AAODS4rUhyWazSZLCwsKc2sPCwhz7bDabQkNDnfb7+vqqY8eOTn3MjvHTc5wvPT1dwcHBji0yMvLiBwQAAJoVrw1JnjRnzhxVVFQ4tuLiYk+XBAAAmpjXhqTw8HBJUklJiVN7SUmJY194eLhKS0ud9p85c0ZlZWVOfcyO8dNznM/f319BQUFOGwAAuLR4bUiKjo5WeHi4Nm3a5Giz2+3KycmR1WqVJFmtVpWXlysvL8/RZ/PmzaqtrVVsbKyjz9atW1VdXe3ok5mZqd69e6tDhw5NNBoAANDceDQkVVZWKj8/X/n5+ZLO3qydn5+vQ4cOyWKxaNq0aXrmmWf097//XXv37tXdd9+tiIgIxzfg+vbtq7Fjx2rKlCnasWOHPvvsM6WmpmrSpEmKiIiQJN15553y8/NTUlKS9u3bpzfffFOLFy/WjBkzPDRqAADQHPh68uQ7d+7UjTfe6Hh/LrhMnjxZK1as0GOPPabjx48rOTlZ5eXluv7667VhwwYFBAQ4PrNy5UqlpqZq9OjR8vHx0cSJE5WRkeHYHxwcrI0bNyolJUVDhgxR586dlZaW5vQsJQAAgPN5zXOSvBnPSQIAoOF4ThIAAEALQkgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAw4XJIOnnypE6cOOF4f/DgQS1atEgbN250a2EAAACe5HJIGj9+vF5//XVJUnl5uWJjY/X8889r/PjxWrZsmdsLBAAA8ASXQ9KuXbs0YsQISdI777yjsLAwHTx4UK+//royMjLcXiAAAIAnuBySTpw4ofbt20uSNm7cqAkTJsjHx0fXXnutDh486PYCAQAAPMHlkNSzZ0+tXbtWxcXF+vjjjzVmzBhJUmlpqYKCgtxeIAAAgCe4HJLS0tI0c+ZMde/eXbGxsbJarZLOrioNHjzY7QUCAAB4gq+rH/jVr36l66+/XkePHtXAgQMd7aNHj9aECRPcWhwAAICnuLySdN999ykwMFCDBw+Wj8//ffyqq67Ss88+69biAAAAPMXlkPTaa6/p5MmTddpPnjzpeDQAAABAc1fvy212u12GYcgwDB07dkwBAQGOfTU1Nfrwww8VGhraKEUCAAA0tXqHpJCQEFksFlksFl155ZV19lssFj311FNuLQ4AAMBT6h2SPv30UxmGoZtuuknvvvuuOnbs6Njn5+enqKgoRURENEqRAAAATa3eIemGG26QJBUVFSkyMtLppm0AAICWxuVHAERFRam8vFw7duxQaWmpamtrnfbffffdbisOAADAU1wOSevWrVNiYqIqKysVFBQki8Xi2GexWAhJAACgRXD5mtkjjzyi++67T5WVlSovL9ePP/7o2MrKyhqjRgAAgCbnckj67rvv9NBDD6lt27aNUQ8AAIBXcDkkxcfHa+fOnY1RCwAAgNdw+Z6khIQEPfroo9q/f7/69++v1q1bO+2/9dZb3VYcAACAp7gckqZMmSJJmjdvXp19FotFNTU1F18VAACAh7kcks7/yj8AAEBLdFFPhDx16pS76gAAAPAqLoekmpoaPf3007r88svVrl07/etf/5IkzZ07V6+88orbCwQAAPAEl0PS73//e61YsUILFiyQn5+fo71fv356+eWX3VocAACAp7gckl5//XW99NJLSkxMVKtWrRztAwcO1JdffunW4mpqajR37lxFR0erTZs26tGjh55++mkZhuHoYxiG0tLS1KVLF7Vp00ZxcXE6cOCA03HKysqUmJiooKAghYSEKCkpSZWVlW6tFQAAtCwNephkz54967TX1taqurraLUWd8+yzz2rZsmX605/+pIKCAj377LNasGCBlixZ4uizYMECZWRkaPny5crJyVFgYKDi4+Od7pdKTEzUvn37lJmZqfXr12vr1q1KTk52a60AAKBlcfnbbTExMfrHP/6hqKgop/Z33nlHgwcPdlthkrRt2zaNHz9eCQkJkqTu3bvrjTfe0I4dOySdXUVatGiRHn/8cY0fP17S2ZWusLAwrV27VpMmTVJBQYE2bNig3NxcDR06VJK0ZMkSjRs3Tn/84x8VERFR57xVVVWqqqpyvLfb7W4dFwAA8H4urySlpaUpNTVVzz77rGpra/Xee+9pypQp+v3vf6+0tDS3Fnfddddp06ZN+uqrryRJn3/+uf75z3/q5ptvliQVFRXJZrMpLi7O8Zng4GDFxsYqOztbkpSdna2QkBBHQJKkuLg4+fj4KCcnx/S86enpCg4OdmyRkZFuHRcAAPB+Loek8ePHa926dfrkk08UGBiotLQ0FRQUaN26dfrFL37h1uJmz56tSZMmqU+fPmrdurUGDx6sadOmKTExUZJks9kkSWFhYU6fCwsLc+yz2WwKDQ112u/r66uOHTs6+pxvzpw5qqiocGzFxcVuHRcAAPB+Ll9uk6QRI0YoMzPT3bXU8dZbb2nlypVatWqVrrrqKuXn52vatGmKiIjQ5MmTG+28/v7+8vf3b7TjAwAA79egkHROZWVlnSdwBwUFXVRBP/Xoo486VpMkqX///jp48KDS09M1efJkhYeHS5JKSkrUpUsXx+dKSko0aNAgSVJ4eLhKS0udjnvmzBmVlZU5Pg8AAHA+ly+3FRUVKSEhQYGBgQoODlaHDh3UoUMHhYSEqEOHDm4t7sSJE/LxcS6xVatWjmAWHR2t8PBwbdq0ybHfbrcrJydHVqtVkmS1WlVeXq68vDxHn82bN6u2tlaxsbFurRcAALQcLq8k3XXXXTIMQ3/9618VFhYmi8XSGHVJkm655Rb9/ve/V7du3XTVVVdp9+7dWrhwoe677z5JZ39Qd9q0aXrmmWfUq1cvRUdHa+7cuYqIiNBtt90mSerbt6/Gjh2rKVOmaPny5aqurlZqaqomTZpk+s02AAAAqQEh6fPPP1deXp569+7dGPU4WbJkiebOnasHHnhApaWlioiI0G9/+1unb9E99thjOn78uJKTk1VeXq7rr79eGzZsUEBAgKPPypUrlZqaqtGjR8vHx0cTJ05URkZGo9cPAACaL4vx08dX18ONN96o3/3ud05fu2/p7Ha7goODVVFR4dZ7rgAAuBR0n/3BBft8Oz/B7ee92L/fLq8kvfzyy5o6daq+++479evXT61bt3baP2DAAJeLAAAA8DYuh6Tvv/9e33zzje69915Hm8VikWEYslgsqqmpcWuBAAAAnuBySLrvvvs0ePBgvfHGG41+4zYAAICnuBySDh48qL///e+mP3ILAADQUrj8nKSbbrpJn3/+eWPUAgAA4DVcXkm65ZZbNH36dO3du1f9+/evc+P2rbfe6rbiAAAAPMXlkDR16lRJ0rx58+rs48ZtAADQUrgcks7/rTYAAICWyOV7kgAAAC4FLq8kmV1m+6mf/mQIAABAc+VySFqzZo3T++rqahUVFcnX11c9evQgJAEAgBbB5ZC0e/fuOm12u1333HOPfvnLX7qlKAAAAE9zyz1JQUFBeuqppzR37lx3HA4AAMDj3HbjdkVFhSoqKtx1OAAAAI9y+XJbRkaG03vDMHT06FH97//+r26++Wa3FQYAAOBJLoekF154wem9j4+PLrvsMk2ePFlz5sxxW2EAAACe5HJIKioqaow6AAAAvIrL9yRVVFSorKysTntZWZnsdrtbigIAAPA0l0PSpEmTtHr16jrtb731liZNmuSWogAAADzN5ZCUk5OjG2+8sU77qFGjlJOT45aiAAAAPM3lkFRVVaUzZ87Uaa+urtbJkyfdUhQAAICnuRyShg0bppdeeqlO+/LlyzVkyBC3FAUAAOBpLn+77ZlnnlFcXJw+//xzjR49WpK0adMm5ebmauPGjW4vEAAAwBNcXkkaPny4srOz1bVrV7311ltat26devbsqT179mjEiBGNUSMAAECTc3klSZIGDRqkVatWubsWAAAAr9GgkFRTU6O1a9eqoKBAknTVVVfp1ltvVatWrdxaHAAAgKe4HJK+/vprJSQk6PDhw+rdu7ckKT09XZGRkfrggw/Uo0cPtxcJAADQ1FwOSQ899JCuuOIKZWdnq2PHjpKkH374QXfddZceeughffDBB24vsqXrPvvC/2bfzk9ogkoAAMA5LoekrKwsbd++3RGQJKlTp06aP3++hg8f7tbiAAAAPMXlb7f5+/vr2LFjddorKyvl5+fnlqIAAAA8zeWQ9F//9V9KTk5WTk6ODMOQYRjavn27pk6dqltvvbUxagQAAGhyLoekjIwM9ejRQ1arVQEBAQoICNDw4cPVs2dPLV68uDFqBAAAaHIu35MUEhKi999/XwcOHNCXX34pSerbt6969uzp9uIAAAA8pUHPSZKkXr16qVevXu6sBQAAwGvUKyTNmDGj3gdcuHBhg4sBAADwFvUKSbt373Z6v2vXLp05c8bxMMmvvvpKrVq10pAhQ9xfIQAAgAfUKyR9+umnjtcLFy5U+/bt9dprr6lDhw6SpB9//FH33nsvP3ALAABaDJe/3fb8888rPT3dEZAkqUOHDnrmmWf0/PPPu7U4AAAAT3E5JNntdn3//fd12r///nvTh0wCAAA0Ry6HpF/+8pe699579d577+nw4cM6fPiw3n33XSUlJWnChAmNUSMAAECTc/kRAMuXL9fMmTN15513qrq6+uxBfH2VlJSk5557zu0FAgAAeILLIalt27Z68cUX9dxzz+mbb76RJPXo0UOBgYFuLw4AAMBTXL7cdk5gYKAGDBigAQMGNGpA+u6773TXXXepU6dOatOmjfr376+dO3c69huGobS0NHXp0kVt2rRRXFycDhw44HSMsrIyJSYmKigoSCEhIUpKSlJlZWWj1QwAAJq/BoekpvDjjz9q+PDhat26tT766CPt379fzz//vNM36xYsWKCMjAwtX75cOTk5CgwMVHx8vE6dOuXok5iYqH379ikzM1Pr16/X1q1blZyc7IkhAQCAZqLBP0vSFJ599llFRkbq1VdfdbRFR0c7XhuGoUWLFunxxx/X+PHjJUmvv/66wsLCtHbtWk2aNEkFBQXasGGDcnNzNXToUEnSkiVLNG7cOP3xj39UREREnfNWVVWpqqrK8d5utzfWEAEAgJfy6pWkv//97xo6dKj++7//W6GhoRo8eLD+8pe/OPYXFRXJZrMpLi7O0RYcHKzY2FhlZ2dLkrKzsxUSEuIISJIUFxcnHx8f5eTkmJ43PT1dwcHBji0yMrKRRggAALxVvULS1VdfrR9//FGSNG/ePJ04caJRizrnX//6l5YtW6ZevXrp448/1v3336+HHnpIr732miTJZrNJksLCwpw+FxYW5thns9kUGhrqtN/X11cdO3Z09DnfnDlzVFFR4diKi4vdPTQAAODl6hWSCgoKdPz4cUnSU0891WQ3PdfW1urqq6/WH/7wBw0ePFjJycmaMmWKli9f3qjn9ff3V1BQkNMGAAAuLfW6J2nQoEG69957df3118swDP3xj39Uu3btTPumpaW5rbguXbooJibGqa1v37569913JUnh4eGSpJKSEnXp0sXRp6SkRIMGDXL0KS0tdTrGmTNnVFZW5vg8AADA+eoVklasWKEnnnhC69evl8Vi0UcffSRf37oftVgsbg1Jw4cPV2FhoVPbV199paioKElnb+IODw/Xpk2bHKHIbrcrJydH999/vyTJarWqvLxceXl5GjJkiCRp8+bNqq2tVWxsrNtqBQAALUu9QlLv3r21evVqSZKPj482bdpU5z6fxjB9+nRdd911+sMf/qDbb79dO3bs0EsvvaSXXnpJ0tlQNm3aND3zzDPq1auXoqOjNXfuXEVEROi2226TdHblaezYsY7LdNXV1UpNTdWkSZNMv9kGAAAgNeARALW1tY1Rh6lrrrlGa9as0Zw5czRv3jxFR0dr0aJFSkxMdPR57LHHdPz4cSUnJ6u8vFzXX3+9NmzYoICAAEeflStXKjU1VaNHj5aPj48mTpyojIyMJhsHAABofiyGYRiufuibb77RokWLVFBQIEmKiYnRww8/rB49eri9QG9gt9sVHBysioqKRrmJu/vsDy7Y59v5CW4/LwAATcFTf+cu9u+3y89J+vjjjxUTE6MdO3Y4fpYkJydHV111lTIzM10uAAAAwBu5fLlt9uzZmj59uubPn1+nfdasWfrFL37htuIAAAA8xeWVpIKCAiUlJdVpv++++7R//363FAUAAOBpLoekyy67TPn5+XXa8/Pzm+QbbwAAAE3B5cttU6ZMUXJysv71r3/puuuukyR99tlnevbZZzVjxgy3FwgAAOAJLoekuXPnqn379nr++ec1Z84cSVJERISefPJJPfTQQ24vEAAAwBNcDkkWi0XTp0/X9OnTdezYMUlS+/bt3V4YAACAJ7kckn6KcAQAAFoql2/cBgAAuBQQkgAAAEwQkgAAAEwQkgAAAEw0KCSlpqaqrKzM3bUAAAB4jXqHpMOHDzter1q1SpWVlZKk/v37q7i42P2VAQAAeFC9HwHQp08fderUScOHD9epU6dUXFysbt266dtvv1V1dXVj1ggAANDk6r2SVF5errfffltDhgxRbW2txo0bpyuvvFJVVVX6+OOPVVJS0ph1AgAANKl6h6Tq6moNGzZMjzzyiNq0aaPdu3fr1VdfVatWrfTXv/5V0dHR6t27d2PWCgAA0GTqfbktJCREgwYN0vDhw3X69GmdPHlSw4cPl6+vr958801dfvnlys3NbcxaAQAAmky9V5K+++47Pf744/L399eZM2c0ZMgQjRgxQqdPn9auXbtksVh0/fXXN2atAAAATabeIalz58665ZZblJ6errZt2yo3N1cPPvigLBaLZs6cqeDgYN1www2NWSsAAECTafDDJIODg3X77berdevW2rx5s4qKivTAAw+4szYAAACPqfc9ST+1Z88eXX755ZKkqKgotW7dWuHh4brjjjvcWhwAAICnNCgkRUZGOl5/8cUXbisGAADAW/DbbQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACYISQAAACaaVUiaP3++LBaLpk2b5mg7deqUUlJS1KlTJ7Vr104TJ05USUmJ0+cOHTqkhIQEtW3bVqGhoXr00Ud15syZJq4eAAA0J80mJOXm5urPf/6zBgwY4NQ+ffp0rVu3Tm+//baysrJ05MgRTZgwwbG/pqZGCQkJOn36tLZt26bXXntNK1asUFpaWlMPAQAANCPNIiRVVlYqMTFRf/nLX9ShQwdHe0VFhV555RUtXLhQN910k4YMGaJXX31V27Zt0/bt2yVJGzdu1P79+/W3v/1NgwYN0s0336ynn35aS5cu1enTpz01JAAA4OWaRUhKSUlRQkKC4uLinNrz8vJUXV3t1N6nTx9169ZN2dnZkqTs7Gz1799fYWFhjj7x8fGy2+3at2+f6fmqqqpkt9udNgAAcGnx9XQBF7J69Wrt2rVLubm5dfbZbDb5+fkpJCTEqT0sLEw2m83R56cB6dz+c/vMpKen66mnnnJD9QAAoLny6pWk4uJiPfzww1q5cqUCAgKa7Lxz5sxRRUWFYysuLm6ycwMAAO/g1SEpLy9PpaWluvrqq+Xr6ytfX19lZWUpIyNDvr6+CgsL0+nTp1VeXu70uZKSEoWHh0uSwsPD63zb7dz7c33O5+/vr6CgIKcNAABcWrw6JI0ePVp79+5Vfn6+Yxs6dKgSExMdr1u3bq1NmzY5PlNYWKhDhw7JarVKkqxWq/bu3avS0lJHn8zMTAUFBSkmJqbJxwQAAJoHr74nqX379urXr59TW2BgoDp16uRoT0pK0owZM9SxY0cFBQXpwQcflNVq1bXXXitJGjNmjGJiYvSb3/xGCxYskM1m0+OPP66UlBT5+/s3+ZgAAEDz4NUhqT5eeOEF+fj4aOLEiaqqqlJ8fLxefPFFx/5WrVpp/fr1uv/++2W1WhUYGKjJkydr3rx5HqwaAAB4u2YXkrZs2eL0PiAgQEuXLtXSpUt/9jNRUVH68MMPG7kyAADQknj1PUkAAACeQkgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAwQUgCAAAw4evpAuC67rM/uGCfb+cnNEElAAC0XKwkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmCAkAQAAmPDqkJSenq5rrrlG7du3V2hoqG677TYVFhY69Tl16pRSUlLUqVMntWvXThMnTlRJSYlTn0OHDikhIUFt27ZVaGioHn30UZ05c6YphwIAAJoZrw5JWVlZSklJ0fbt25WZmanq6mqNGTNGx48fd/SZPn261q1bp7fffltZWVk6cuSIJkyY4NhfU1OjhIQEnT59Wtu2bdNrr72mFStWKC0tzRNDAgAAzYSvpwv4TzZs2OD0fsWKFQoNDVVeXp5GjhypiooKvfLKK1q1apVuuukmSdKrr76qvn37avv27br22mu1ceNG7d+/X5988onCwsI0aNAgPf3005o1a5aefPJJ+fn5eWJoAADAy3n1StL5KioqJEkdO3aUJOXl5am6ulpxcXGOPn369FG3bt2UnZ0tScrOzlb//v0VFhbm6BMfHy+73a59+/aZnqeqqkp2u91pAwAAl5ZmE5Jqa2s1bdo0DR8+XP369ZMk2Ww2+fn5KSQkxKlvWFiYbDabo89PA9K5/ef2mUlPT1dwcLBji4yMdPNoAACAt2s2ISklJUVffPGFVq9e3ejnmjNnjioqKhxbcXFxo58TAAB4F6++J+mc1NRUrV+/Xlu3blXXrl0d7eHh4Tp9+rTKy8udVpNKSkoUHh7u6LNjxw6n45379tu5Pufz9/eXv7+/m0cBAACaE69eSTIMQ6mpqVqzZo02b96s6Ohop/1DhgxR69attWnTJkdbYWGhDh06JKvVKkmyWq3au3evSktLHX0yMzMVFBSkmJiYphkIAABodrx6JSklJUWrVq3S+++/r/bt2zvuIQoODlabNm0UHByspKQkzZgxQx07dlRQUJAefPBBWa1WXXvttZKkMWPGKCYmRr/5zW+0YMEC2Ww2Pf7440pJSWG1CAAA/CyvDknLli2TJI0aNcqp/dVXX9U999wjSXrhhRfk4+OjiRMnqqqqSvHx8XrxxRcdfVu1aqX169fr/vvvl9VqVWBgoCZPnqx58+Y11TAAAEAz5NUhyTCMC/YJCAjQ0qVLtXTp0p/tExUVpQ8//NCdpQEAgBbOq+9JAgAA8BRCEgAAgAmvvtyGi9d99gcX7PPt/IQmqAQAgOaFlSQAAAAThCQAAAAThCQAAAAThCQAAAAThCQAAAAThCQAAAAThCQAAAAThCQAAAAThCQAAAAThCQAAAAT/CwJnPAzJgAAnMVKEgAAgAlCEgAAgAlCEgAAgAlCEgAAgAlCEgAAgAlCEgAAgAlCEgAAgAmek4QGq88zlSSeqwQAaJ5YSQIAADBBSAIAADBBSAIAADDBPUloMvwuHACgOWElCQAAwAQhCQAAwASX2+C1uDwHAPAkVpIAAABMsJKEFsHVB1uySgUAuBBWkgAAAEwQkgAAAExwuQ2oBy7PAcClh5AEuJmrgYoABgDeiZAENCPcoA4ATYeQBMCBUAUA/4eQBKBBXF3VAoDmhm+3AQAAmGAlCUCT4FIegOaGkATAK7kSqhrzG4VcVgQuXYQkAHAjbwps3nLsxq4FaCyXVEhaunSpnnvuOdlsNg0cOFBLlizRsGHDPF0WAOAieFNgIwy2LJdMSHrzzTc1Y8YMLV++XLGxsVq0aJHi4+NVWFio0NBQT5cHAMB/1FzDYHN2yXy7beHChZoyZYruvfdexcTEaPny5Wrbtq3++te/ero0AADghS6JlaTTp08rLy9Pc+bMcbT5+PgoLi5O2dnZdfpXVVWpqqrK8b6iokKSZLfbG6W+2qoTF+zz03O70t/Tx6aWpj82tTT9sVtCLZfKOL2plktlnA3p7y7njmkYRsMOYFwCvvvuO0OSsW3bNqf2Rx991Bg2bFid/k888YQhiY2NjY2Nja0FbMXFxQ3KD5fESpKr5syZoxkzZjje19bWqqysTJ06dZLFYmn089vtdkVGRqq4uFhBQUGNfj5PYZwtC+NsWRhny3KpjtMwDB07dkwRERENOt4lEZI6d+6sVq1aqaSkxKm9pKRE4eHhdfr7+/vL39/fqS0kJKQxSzQVFBTUov/HfA7jbFkYZ8vCOFuWS3GcwcHBDT7OJXHjtp+fn4YMGaJNmzY52mpra7Vp0yZZrVYPVgYAALzVJbGSJEkzZszQ5MmTNXToUA0bNkyLFi3S8ePHde+993q6NAAA4IUumZB0xx136Pvvv1daWppsNpsGDRqkDRs2KCwszNOl1eHv768nnniiziW/loZxtiyMs2VhnC0L42wYi2E09HtxAAAALdclcU8SAACAqwhJAAAAJghJAAAAJghJAAAAJghJXmbp0qXq3r27AgICFBsbqx07dni6JLd68sknZbFYnLY+ffp4uiy32Lp1q2655RZFRETIYrFo7dq1TvsNw1BaWpq6dOmiNm3aKC4uTgcOHPBMsRfhQuO855576szx2LFjPVNsA6Wnp+uaa65R+/btFRoaqttuu02FhYVOfU6dOqWUlBR16tRJ7dq108SJE+s8sNbb1Weco0aNqjOfU6dO9VDFDbds2TINGDDA8ZBBq9Wqjz76yLG/JcyndOFxtpT5/Kn58+fLYrFo2rRpjjZ3zSchyYu8+eabmjFjhp544gnt2rVLAwcOVHx8vEpLSz1dmltdddVVOnr0qGP75z//6emS3OL48eMaOHCgli5darp/wYIFysjI0PLly5WTk6PAwEDFx8fr1KlTTVzpxbnQOCVp7NixTnP8xhtvNGGFFy8rK0spKSnavn27MjMzVV1drTFjxuj48eOOPtOnT9e6dev09ttvKysrS0eOHNGECRM8WLXr6jNOSZoyZYrTfC5YsMBDFTdc165dNX/+fOXl5Wnnzp266aabNH78eO3bt09Sy5hP6cLjlFrGfJ6Tm5urP//5zxowYIBTu9vms0G/+IZGMWzYMCMlJcXxvqamxoiIiDDS09M9WJV7PfHEE8bAgQM9XUajk2SsWbPG8b62ttYIDw83nnvuOUdbeXm54e/vb7zxxhseqNA9zh+nYRjG5MmTjfHjx3uknsZSWlpqSDKysrIMwzg7d61btzbefvttR5+CggJDkpGdne2pMi/a+eM0DMO44YYbjIcffthzRTWiDh06GC+//HKLnc9zzo3TMFrWfB47dszo1auXkZmZ6TQud84nK0le4vTp08rLy1NcXJyjzcfHR3FxccrOzvZgZe534MABRURE6IorrlBiYqIOHTrk6ZIaXVFRkWw2m9P8BgcHKzY2tsXNryRt2bJFoaGh6t27t+6//3798MMPni7polRUVEiSOnbsKEnKy8tTdXW103z26dNH3bp1a9bzef44z1m5cqU6d+6sfv36ac6cOTpx4oQnynObmpoarV69WsePH5fVam2x83n+OM9pKfOZkpKihIQEp3mT3Pvf5yXzxG1v9+9//1s1NTV1ngAeFhamL7/80kNVuV9sbKxWrFih3r176+jRo3rqqac0YsQIffHFF2rfvr2ny2s0NptNkkzn99y+lmLs2LGaMGGCoqOj9c033+h//ud/dPPNNys7O1utWrXydHkuq62t1bRp0zR8+HD169dP0tn59PPzq/PD1815Ps3GKUl33nmnoqKiFBERoT179mjWrFkqLCzUe++958FqG2bv3r2yWq06deqU2rVrpzVr1igmJkb5+fktaj5/bpxSy5nP1atXa9euXcrNza2zz53/fRKS0KRuvvlmx+sBAwYoNjZWUVFReuutt5SUlOTByuAukyZNcrzu37+/BgwYoB49emjLli0aPXq0BytrmJSUFH3xxRct5t65n/Nz40xOTna87t+/v7p06aLRo0frm2++UY8ePZq6zIvSu3dv5efnq6KiQu+8844mT56srKwsT5fldj83zpiYmBYxn8XFxXr44YeVmZmpgICARj0Xl9u8ROfOndWqVas6d9+XlJQoPDzcQ1U1vpCQEF155ZX6+uuvPV1Kozo3h5fa/ErSFVdcoc6dOzfLOU5NTdX69ev16aefqmvXro728PBwnT59WuXl5U79m+t8/tw4zcTGxkpSs5xPPz8/9ezZU0OGDFF6eroGDhyoxYsXt7j5/LlxmmmO85mXl6fS0lJdffXV8vX1la+vr7KyspSRkSFfX1+FhYW5bT4JSV7Cz89PQ4YM0aZNmxxttbW12rRpk9O15JamsrJS33zzjbp06eLpUhpVdHS0wsPDnebXbrcrJyenRc+vJB0+fFg//PBDs5pjwzCUmpqqNWvWaPPmzYqOjnbaP2TIELVu3dppPgsLC3Xo0KFmNZ8XGqeZ/Px8SWpW8/lzamtrVVVV1WLm8+ecG6eZ5jifo0eP1t69e5Wfn+/Yhg4dqsTERMdrt82n++4zx8VavXq14e/vb6xYscLYv3+/kZycbISEhBg2m83TpbnNI488YmzZssUoKioyPvvsMyMuLs7o3LmzUVpa6unSLtqxY8eM3bt3G7t37zYkGQsXLjR2795tHDx40DAMw5g/f74REhJivP/++8aePXuM8ePHG9HR0cbJkyc9XLlr/tM4jx07ZsycOdPIzs42ioqKjE8++cS4+uqrjV69ehmnTp3ydOn1dv/99xvBwcHGli1bjKNHjzq2EydOOPpMnTrV6Natm7F582Zj586dhtVqNaxWqwerdt2Fxvn1118b8+bNM3bu3GkUFRUZ77//vnHFFVcYI0eO9HDlrps9e7aRlZVlFBUVGXv27DFmz55tWCwWY+PGjYZhtIz5NIz/PM6WNJ/nO/9be+6aT0KSl1myZInRrVs3w8/Pzxg2bJixfft2T5fkVnfccYfRpUsXw8/Pz7j88suNO+64w/j66689XZZbfPrpp4akOtvkyZMNwzj7GIC5c+caYWFhhr+/vzF69GijsLDQs0U3wH8a54kTJ4wxY8YYl112mdG6dWsjKirKmDJlSrML+mbjk2S8+uqrjj4nT540HnjgAaNDhw5G27ZtjV/+8pfG0aNHPVd0A1xonIcOHTJGjhxpdOzY0fD39zd69uxpPProo0ZFRYVnC2+A++67z4iKijL8/PyMyy67zBg9erQjIBlGy5hPw/jP42xJ83m+80OSu+bTYhiG0cAVLwAAgBaLe5IAAABMEJIAAABMEJIAAABMEJIAAABMEJIAAABMEJIAAABMEJIAAABMEJIAAABMEJIAXNK6d++uRYsWeboMAF6IkATA640aNUrTpk1rlGPn5uYqOTnZLcd6++23dd1110mStm3bpiuuuMItxwXgGb6eLgAAPOmyyy5z27Gys7M1fPhwSdI//vEPx2sAzRMrSQC82j333KOsrCwtXrxYFotFFotF3377rSQpKytLw4YNk7+/v7p06aLZs2frzJkzjs+OGjVKqampSk1NVXBwsDp37qy5c+fqpz9Zef7ltvLycv32t79VWFiYAgIC1K9fP61fv75etW7bts0RjP75z38SkoBmjpAEwKstXrxYVqtVU6ZM0dGjR3X06FFFRkbqu+++07hx43TNNdfo888/17Jly/TKK6/omWeecfr8a6+9Jl9fX+3YsUOLFy/WwoUL9fLLL5ueq7a2VjfffLM+++wz/e1vf9P+/fs1f/58tWrV6mfrW7VqlUJCQhQSEqIdO3boN7/5jUJCQvThhx9q5syZCgkJ0apVq9z6bwKgaXC5DYBXCw4Olp+fn9q2bavw8HBH+4svvqjIyEj96U9/ksViUZ8+fXTkyBHNmjVLaWlp8vE5+/8BIyMj9cILL8hisah3797au3evXnjhBU2ZMqXOuT755BPt2LFDBQUFuvLKKyXpgvcV3Xrrrbruuuv0ySefaNGiRVq/fr327NmjqVOnatu2bZKkzp07u+ufA0ATYiUJQLNUUFAgq9Uqi8XiaBs+fLgqKyt1+PBhR9u1117r1MdqterAgQOqqampc8z8/Hx17drVEZDqo127durevbt27dql8ePHq3v37tq7d6/GjRun7t27q3v37mrXrl0DRwnAk1hJAoD/r02bNi71P3TokGJiYiRJp06dkq+vrxYvXqyqqir5+Pho9erVuuuuu7R8+fLGKBdAIyMkAfB6fn5+dVZ++vbtq3fffVeGYThWij777DO1b99eXbt2dfTLyclx+tz27dvVq1cv0/uMBgwYoMOHD+urr76q12pSRESE8vPzZbPZFBcXp/z8fNXU1GjQoEH6xz/+oY4dOyooKKghQwbgBbjcBsDrde/eXTk5Ofr222/173//W7W1tXrggQdUXFysBx98UF9++aXef/99PfHEE5oxY4bjfiTp7GrPjBkzVFhYqDfeeENLlizRww8/bHqeG264QSNHjtTEiROVmZmpoqIiffTRR9qwYYNpf19fX/Xs2VOHDx9WbGys+vTpox9++EFXXHGFhg0bpp49eyo0NLRR/k0AND5CEgCvN3PmTLVq1UoxMTG67LLLdOjQIV1++eX68MMPtWPHDg0cOFBTp05VUlKSHn/8cafP3n333Tp58qSGDRumlJQUPfzww//x4ZHvvvuurrnmGv36179WTEyMHnvsMdP7l35qy5YtGjlypKSzjyU49xpA82YxfvrAEABoQUaNGqVBgwbxsyMAGoSVJAAAABOEJAAAABNcbgMAADDBShIAAIAJQhIAAIAJQhIAAIAJQhIAAIAJQhIAAIAJQhIAAIAJQhIAAIAJQhIAAICJ/we+/WtIzzW52AAAAABJRU5ErkJggg==\n"},"metadata":{}}]},{"cell_type":"code","source":["# extract measures\n","NMI = nmi_fn(Pwc)\n","Q = modularity_fn(Pcc)\n","Ncut = ncut_fn(Pcc)\n","rd = infomap_rank_fn(in_data.Pdd) # we need the PageRank vector first\n","Infomap = infomap_fn(C,in_data.Pdd,rd)\n","if (pc.shape[1]==1):\n"," com = 0\n","else:\n"," com = _infomap_fn(pc)/np.log(pc.shape[1])"],"metadata":{"id":"Wo4Ytt-DkOFT","executionInfo":{"status":"ok","timestamp":1765745033493,"user_tz":-60,"elapsed":3046,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}}},"execution_count":41,"outputs":[]},{"cell_type":"code","source":["# collect them in dataframe\n","pd.DataFrame(data = {'topics': C.shape[1], 'com': com,\n"," 'NMI': NMI, 'Q': Q, 'Ncut': Ncut, 'Infomap': Infomap}, index=[0])"],"metadata":{"id":"PY7lCxvqrJW1","colab":{"base_uri":"https://localhost:8080/","height":81},"executionInfo":{"status":"ok","timestamp":1765745034534,"user_tz":-60,"elapsed":16,"user":{"displayName":"Tomaso Erseghe","userId":"15955126948488574654"}},"outputId":"ee39d3b3-0263-4973-89ee-6a683e7c29fc"},"execution_count":42,"outputs":[{"output_type":"execute_result","data":{"text/plain":[" topics com NMI Q Ncut Infomap\n","0 39 0.664808 0.29315 0.096722 0.857763 0.085053"],"text/html":["\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
topicscomNMIQNcutInfomap
0390.6648080.293150.0967220.8577630.085053
\n","
\n","
\n","\n","
\n"," \n","\n"," \n","\n"," \n","
\n","\n","\n","
\n","
\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","summary":"{\n \"name\": \" 'NMI': NMI, 'Q': Q, 'Ncut': Ncut, 'Infomap': Infomap}, index=[0])\",\n \"rows\": 1,\n \"fields\": [\n {\n \"column\": \"topics\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": null,\n \"min\": 39,\n \"max\": 39,\n \"num_unique_values\": 1,\n \"samples\": [\n 39\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"com\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": null,\n \"min\": 0.6648078888653084,\n \"max\": 0.6648078888653084,\n \"num_unique_values\": 1,\n \"samples\": [\n 0.6648078888653084\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"NMI\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": null,\n \"min\": 0.2931497725659325,\n \"max\": 0.2931497725659325,\n \"num_unique_values\": 1,\n \"samples\": [\n 0.2931497725659325\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Q\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": null,\n \"min\": 0.09672155125395793,\n \"max\": 0.09672155125395793,\n \"num_unique_values\": 1,\n \"samples\": [\n 0.09672155125395793\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Ncut\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": null,\n \"min\": 0.857762661569703,\n \"max\": 0.857762661569703,\n \"num_unique_values\": 1,\n \"samples\": [\n 0.857762661569703\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Infomap\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": null,\n \"min\": 0.08505316873891888,\n \"max\": 0.08505316873891888,\n \"num_unique_values\": 1,\n \"samples\": [\n 0.08505316873891888\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"}},"metadata":{},"execution_count":42}]}],"metadata":{"colab":{"gpuType":"T4","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":0}