Tests pass on new database schema

Author: elsehow

INSTALL.md

@@ -31,6 +31,7 @@ Then, if you want to restore from a database backup:
 psql -U scraper -d taaraxtak -h localhost < [your-database-dump].sql 
 ```
 
+Alternatively, you can initiate a fresh database.  See Python setup, below.
 
 ### Python
 
@@ -48,7 +49,7 @@ pip3 install -e .
 
 3. Copy `config.example.py` to `config.py` and enter your Postgres credentials. See Config section below for info on
    logging options.
-4. Run `python3 create_tables.py` to set up the database. (Alternatively, restore the database from a recent DB dump).
+4. Run `python3 create_tables.py` to set up the database. (Alternatively, restore the database from a recent DB dump. See above.)
 
 Now you can run `python3 collect.py` to start collecting data.
 

README.md

@@ -22,6 +22,13 @@ For local development, start the collection server with
 python3 collect.py
 ```
 
+You can also run the data collection as separate one-time jobs:
+
+- `python3 run.py w3techs`
+- `python3 run.py ooni`
+
+This makes it suitable to be run from a system cron, rather than as a standalone continuous process.
+
 Then check out your Grafana instance (by default, https://localhost:3000).
 
 To deploy a production environment, see [DEPLOY.md](DEPLOY.md)

create_tables.py

@@ -31,5 +31,5 @@
 #
 # run
 #
-# w3techs()
-# ooni()
+w3techs()
+ooni()

src/w3techs/collect.py

@@ -132,7 +132,7 @@ def collect(postgres_config: dict):
     # Compute gini coefficients
     for market in included_markets:
         logging.info(f'Computing gini for {market}')
-        pop_weighted_gini = utils.population_weighted_gini(cur, market, pd.Timestamp(datetime.now()))
+        pop_weighted_gini = utils.population_weighted_gini(cur, 'all', market, pd.Timestamp(datetime.now()))
         pop_weighted_gini.write_to_db(cur, conn)
 
     logging.debug('W3Techs complete.')

src/w3techs/types.py

@@ -31,6 +31,7 @@ def __init__(self,
                  name: str,
                  url: Optional[str],
                  jurisdiction_alpha2: Optional[shared_types.Alpha2],
+                 measurement_scope: str,
                  market: str,
                  marketshare: float,
                  time: pd.Timestamp):
@@ -50,6 +51,13 @@ def __init__(self,
             # we'll just store the str version
             self.jurisdiction_alpha2 = str(jurisdiction_alpha2)
 
+        assert(
+            (measurement_scope == 'all') or
+            (measurement_scope == '10k') or
+            (measurement_scope == '1k')
+        )
+        self.measurement_scope = measurement_scope
+
         assert(shared_utils.is_nonempty_str(market))
         self.market = market
 
@@ -68,6 +76,7 @@ def create_table(
         name                VARCHAR NOT NULL,
         url                 VARCHAR,
         jurisdiction_alpha2 CHAR(2),
+        measurement_scope   VARCHAR NOT NULL,
         market              VARCHAR NOT NULL,
         marketshare         NUMERIC NOT NULL,
         time                TIMESTAMPTZ NOT NULL DEFAULT now()
@@ -85,12 +94,13 @@ def write_to_db(
         cur.execute(
             """
             INSERT INTO provider_marketshare
-            (name, url, jurisdiction_alpha2, market, marketshare, time)
+            (name, url, jurisdiction_alpha2, measurement_scope, market, marketshare, time)
             VALUES
-            (%s, %s, %s, %s, %s, %s)
+            (%s, %s, %s, %s, %s, %s, %s)
             """, (self.name,
                   self.url,
                   self.jurisdiction_alpha2,
+                  self.measurement_scope,
                   self.market,
                   self.marketshare,
                   self.time))
@@ -99,7 +109,7 @@ def write_to_db(
         return
 
     def __str__(self):
-        return f'{self.name} {self.url} {self.jurisdiction_alpha2}  {self.market} {self.marketshare} {self.time}'
+        return f'{self.name} {self.url} {self.jurisdiction_alpha2} {self.measurement_scope}   {self.market} {self.marketshare} {self.time}'
 
     def __repr__(self):
         return self.__str__()
@@ -114,12 +124,20 @@ class PopWeightedGini ():
     TODO - Check for SQL injection attacks.
     '''
     def __init__(self,
+                 measurement_scope: str,
                  market: str,
                  gini: float,
                  time: pd.Timestamp):
         assert(shared_utils.is_nonempty_str(market))
         self.market = market
 
+        assert(
+            (measurement_scope == 'all') or
+            (measurement_scope == '10k') or
+            (measurement_scope == '1k')
+        )
+        self.measurement_scope = measurement_scope
+
         assert(is_float_0_1(float(gini)))
         self.gini = gini
 
@@ -132,6 +150,7 @@ def create_table(
             conn: connection):
         cmd = '''
         CREATE TABLE pop_weighted_gini (
+        measurement_scope   VARCHAR NOT NULL,
         market              VARCHAR NOT NULL,
         gini                NUMERIC NOT NULL,
         time                TIMESTAMPTZ NOT NULL DEFAULT now()
@@ -149,16 +168,16 @@ def write_to_db(
         cur.execute(
             """
             INSERT INTO pop_weighted_gini
-            (market, gini, time)
+            (measurement_scope, market, gini, time)
             VALUES
-            (%s, %s, %s)
-            """, (self.market, self.gini, self.time))
+            (%s, %s, %s, %s)
+            """, (self.measurement_scope, self.market, self.gini, self.time))
         if commit:
             return conn.commit()
         return
 
     def __str__(self):
-        return f'{self.market} {self.gini} {self.time}'
+        return f'{self.measurement_scope} {self.market} {self.gini} {self.time}'
 
     def __repr__(self):
         return self.__str__()
@@ -171,10 +190,10 @@ def create_tables(cur: cursor, conn: connection):
 
     # dummy data - just a demo
     ProviderMarketshare(
-        'name', None, shared_types.Alpha2('CA'), 'ssl-certificate', 0.5, pd.Timestamp('2021-04-20')
+        'name', None, shared_types.Alpha2('CA'), 'all', 'ssl-certificate', 0.5, pd.Timestamp('2021-04-20')
     ).create_table(cur, conn)
 
     # dummy data - just a demo
     PopWeightedGini(
-        'ssl-certificate', 0.9, pd.Timestamp('2021-04-20')
+        'all', 'ssl-certificate', 0.9, pd.Timestamp('2021-04-20')
     ).create_table(cur, conn)

src/w3techs/utils.py

@@ -130,7 +130,7 @@ def extract_from_row(market: str, time: pd.Timestamp, df_row: pd.Series) -> Prov
     # Once data is in this type, it *should* be trustworthy.
     # See /design-notes.md for more detail on this pattern.
     return ProviderMarketshare(
-        str(name), str(url), juris, market, float(marketshare), time
+        str(name), str(url), juris, 'all', market, float(marketshare), time
     )
 
 
@@ -144,25 +144,26 @@ def extract_from_row(market: str, time: pd.Timestamp, df_row: pd.Series) -> Prov
 
 def to_df(db_rows) -> pd.DataFrame:
     # TODO put this in TYPES somehow?
-    db_rows = pd.DataFrame(db_rows, columns=['name', 'url', 'jurisdiction_alpha2', 'market', 'marketshare', 'time'])
+    db_rows = pd.DataFrame(db_rows, columns=['name', 'url', 'jurisdiction_alpha2', 'measurement_scope', 'market', 'marketshare', 'time'])
     return db_rows
 
 
-def fetch_rows(cur: cursor, market: str, date: pd.Timestamp) -> pd.DataFrame:
+def fetch_rows(cur: cursor, measurement_scope: str, market: str,  date: pd.Timestamp) -> pd.DataFrame:
     # TODO why this window? a magic number.
     cur.execute(f'''
         SELECT * from provider_marketshare
-        WHERE market = '{market}'
+        WHERE measurement_scope = '{measurement_scope}'
+        AND market = '{market}'
         AND time BETWEEN timestamp '{date}' - interval '24 hour' AND  '{date}'
     ''')
     return to_df(cur.fetchall())
 
 
-def fetch_by_jurisdiction(cur: cursor, market: str, date: pd.Timestamp) -> pd.DataFrame:
+def fetch_by_jurisdiction(cur: cursor, measurement_scope: str, market: str,  date: pd.Timestamp) -> pd.DataFrame:
     '''
     Get a DataFrame mapping alpha2 codes to (mean) marketshares on a given date.
     '''
-    rows = fetch_rows(cur, market, date)
+    rows = fetch_rows(cur, measurement_scope, market, date)
     rows['marketshare'] = rows['marketshare'].astype(float)
     # in case we have the same name and jurisidiction repeated, we take the median marketshare
     rows = rows.groupby(['name', 'jurisdiction_alpha2']).median()
@@ -206,8 +207,8 @@ def weighted_gini(marketshares: pd.Series, population_shares: pd.Series) -> floa
     return gini(vs)
 
 
-def population_weighted_gini(cur: cursor, market: str, time: pd.Timestamp) -> Optional[PopWeightedGini]:
-    by_juris = fetch_by_jurisdiction(cur, market, time)
+def population_weighted_gini(cur: cursor, measurement_scope: str, market: str, time: pd.Timestamp) -> Optional[PopWeightedGini]:
+    by_juris = fetch_by_jurisdiction(cur, measurement_scope, market, time)
     # if there are no values, None
     if len(by_juris) == 0:
         return None
@@ -228,4 +229,4 @@ def population_weighted_gini(cur: cursor, market: str, time: pd.Timestamp) -> Op
         merged['marketshare'],
         merged[relevant_year],
     )
-    return PopWeightedGini(market, g, time)
+    return PopWeightedGini(measurement_scope, market, g, time)

test/w3techs-test.py

@@ -68,7 +68,7 @@ def test_provider_marketshare_type(postgresdb):
     cur, conn = postgresdb
 
     ex_ms = types.ProviderMarketshare(
-        'Foo', None, shared_types.Alpha2('NL'), 'ssl-certificate', 0.5, pd.Timestamp('2021-04-20')
+        'Foo', None, shared_types.Alpha2('NL'), 'all', 'ssl-certificate', 0.5, pd.Timestamp('2021-04-20')
     )
     ex_ms.write_to_db(cur, conn)
 
@@ -81,14 +81,14 @@ def test_pop_weighted_gini_type(postgresdb):
     cur, conn = postgresdb
 
     ex_g = types.PopWeightedGini(
-        'ssl-certificate', 0.9, pd.Timestamp('2021-04-20')
+        'all', 'ssl-certificate', 0.9, pd.Timestamp('2021-04-20')
     )
 
     ex_g.write_to_db(cur, conn)
 
     cur.execute('SELECT * FROM pop_weighted_gini')
     item = cur.fetchone()
-    assert(item[0] == 'ssl-certificate')
+    assert(item[1] == 'ssl-certificate')
 
 #
 #  utils tests
@@ -128,6 +128,7 @@ def test_compute_pop_weighted_gini(postgresdb):
     cur, conn = postgresdb
     res = utils.population_weighted_gini(
         cur,
+        'all',
         'fake-market',
         pd.Timestamp('2021-01-20'),
     )
@@ -136,17 +137,17 @@ def test_compute_pop_weighted_gini(postgresdb):
     # add a provider marketshare
     # tiny netherlands has 50% of the world's market
     types.ProviderMarketshare(
-        'Foo', None, shared_types.Alpha2('NL'), 'ssl-certificate',
+        'Foo', None, shared_types.Alpha2('NL'), 'all', 'ssl-certificate',
         0.5, pd.Timestamp('2021-04-20')
     ).write_to_db(cur, conn)
     # US has the rest
     types.ProviderMarketshare(
-        'Foo', None, shared_types.Alpha2('US'), 'ssl-certificate',
+        'Foo', None, shared_types.Alpha2('US'), 'all', 'ssl-certificate',
         0.5, pd.Timestamp('2021-04-20')
     ).write_to_db(cur, conn)
 
     res = utils.population_weighted_gini(
-        cur, 'ssl-certificate', pd.Timestamp('2021-04-20')
+        cur, 'all', 'ssl-certificate', pd.Timestamp('2021-04-20')
     )
     # should result in a gini of 0.99
     assert(round(res.gini, 2) == 0.99)