#Python script
#This will load all data as a Pandas dataframe
#courtesy of Abyrint.com
#MIT license
import pandas as pd
ep = endpoint //get endpoint from the tab
#reads the index api
tableType = 'dbSomaliaFgsRevExp' #customize this to the type of table required 
#reads the indexfile
idx = pd.read_csv(ep,sep='\t', header=0)
url_list =[]
for index, rows in idx.iterrows():
if rows.standardName == tableType: #change for alternative conditions
my_list =rows.tsv
url_list.append(my_list)
#loads all
dfs = [pd.read_csv(url,sep='\t', header=0) for url in url_list]
df = pd.concat(dfs, ignore_index=True)
#Do analysis with the compiled datatable here
print(df.shape)
     
                                    

//Javascript, Node.js
//This will compile data for all years available as an array of objects in JSON format.
//Each object in the array holds key/value pairs that matches the headers. 
//courtesy of Abyrint.com
// MIT license
const url = endpoint //get url from endpoint tab
const fetch = require('node-fetch');
const tableType = 'dbSomaliaFgsRevExp' //customize this to the type of table required
//const year = '2021' //use this for alternative filter
//builds compiled JSON object for all tables of this type
bigArr = []
const dataset = getData(url).then(x => {
var index = tToJson(x)
index.forEach(function (row) {
if (row.standardName == tableType) { // && row.year >= year //add to filter if required
    getData(row.tsv).then(y => {
        var data = tToJson(y)
        data.forEach(function (q) { bigArr.push(q) })
        console.log(bigArr.length)
        // Do anything with the dataset here
        // bigArr
    })
}
})
})

function getData(url) {
try {
return fetch(url).then(res => { return res.text() }).then(body => {return body})
} catch (err) {
alert(err);
}
}
//TSV parser to JSON
function tToJson(str) {
const lines = str.replace(/\r\n?/g, "\n").split("\n");
const headers = lines.shift().split('\t');
return lines.map(line => {
const data = line.split('\t');
return headers.reduce((o, k, i) => {
    o[k] = data[i];
    return o;
}, {});
});
}                 
                                                            

const endpoint = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vSAu1M-o9hgbZIKlvliDeqXle1LHWN82Li0xPrpzkudj1ACKLsgoIKESgQ5Si7fSAwwrlFIgU0kHkun/pub?output=tsv'