Skip to content

Commit ce14975

Browse files
unyticsunytics
committed
doc: generate doc for tables in data folder
1 parent f25c486 commit ce14975

File tree

11 files changed

+1266
-30
lines changed

11 files changed

+1266
-30
lines changed

‎bigfun/generate_doc.py

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
TEMPLATE_MKDOCS_YAML = TEMPLATE_FOLDER + '/mkdocs.yml'
1010
DOCUMENTATION_CONFIG_FILENAME = 'config_documentation.yaml'
1111
BIGFUNCTIONS_FOLDER = 'bigfunctions'
12+
TABLES_FOLDER = 'data'
1213

1314
if not os.path.isfile('README.md'):
1415
print('INFO: CREATING A README.md FILE IN CURRENT DIRECTORY WHICH WILL BE THE ROOT CONTENT OF THE WEBSITE')
@@ -20,18 +21,31 @@
2021

2122

2223
def get_bigfunctions():
23-
bigfunctions = {}
24+
bigfunctions = []
2425
for filename in sorted(os.listdir(BIGFUNCTIONS_FOLDER)):
2526
if not filename.endswith('.yaml'):
2627
continue
27-
conf = yaml.safe_load(open(f'bigfunctions/{filename}', encoding='utf-8').read())
28+
conf = yaml.safe_load(open(f'{BIGFUNCTIONS_FOLDER}/{filename}', encoding='utf-8').read())
2829
assert isinstance(conf, dict), f'Could not load yaml config of bigfunction `{filename}`'
2930
name = filename.replace('.yaml', '')
3031
conf['name'] = name
31-
bigfunctions[name] = conf
32+
bigfunctions.append(conf)
3233
return bigfunctions
3334

3435

36+
def get_tables():
37+
tables = []
38+
for filename in sorted(os.listdir(TABLES_FOLDER)):
39+
if not filename.endswith('.yaml'):
40+
continue
41+
conf = yaml.safe_load(open(f'{TABLES_FOLDER}/{filename}', encoding='utf-8').read())
42+
assert isinstance(conf, dict), f'Could not load yaml config of table `{filename}`'
43+
name = filename.replace('.yaml', '')
44+
conf['name'] = name
45+
tables.append(conf)
46+
return tables
47+
48+
3549
# def get_documentation_config():
3650
# if not os.path.isfile(DOCUMENTATION_CONFIG_FILENAME):
3751
# print(f'WARNING: Could not find file `{DOCUMENTATION_CONFIG_FILENAME}` in current directory.')
@@ -43,6 +57,7 @@ def get_bigfunctions():
4357

4458

4559
BIGFUNCTIONS = get_bigfunctions()
60+
TABLES = get_tables()
4661
# DOC_CONFIG = get_documentation_config()
4762

4863

@@ -52,35 +67,47 @@ def get_bigfunctions():
5267

5368
class BasePage:
5469

70+
name = ''
5571

5672
def generate(self):
5773
context = self.get_context()
58-
template = f'{TEMPLATE_FOLDER}/{self.template_filename}'
74+
template = f'{TEMPLATE_FOLDER}/{self.name}.md'
5975
content = jinja2.Template(open(template, encoding='utf-8').read()).render(**context)
60-
with open(f'bigfunctions/README.md', 'w', encoding='utf-8') as out:
76+
with open(f'{self.name}/README.md', 'w', encoding='utf-8') as out:
6177
out.write(content)
6278

6379
def get_context(self):
6480
raise NotImplementedError()
6581

6682

67-
class ReferencePage(BasePage):
83+
class BigFunctionsPage(BasePage):
6884

69-
template_filename = 'bigfunctions.md'
85+
name = 'bigfunctions'
7086

7187
def get_context(self):
7288
categories = CONF['bigfunctions_categories']
7389
for category in categories:
74-
category['bigfunctions'] = [bigfunction for bigfunction in BIGFUNCTIONS.values() if bigfunction['category'] == category['name']]
90+
category['bigfunctions'] = [bigfunction for bigfunction in BIGFUNCTIONS if bigfunction['category'] == category['name']]
7591
return {
7692
'datasets': CONF['bigfunctions_datasets'],
7793
'repo_url': CONF['repo_url'],
7894
'categories': categories,
7995
}
8096

97+
class TablesPage(BasePage):
98+
99+
name = 'data'
100+
101+
def get_context(self):
102+
return {
103+
'tables': TABLES,
104+
'repo_url': CONF['repo_url'],
105+
}
106+
81107

82108
def generate_doc():
83-
ReferencePage().generate()
109+
BigFunctionsPage().generate()
110+
TablesPage().generate()
84111

85112

86113
if __name__ == '__main__':

‎bigfun/templates/bigfunctions.md

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,57 @@ description: "Catalog of open-source BigFunctions"
55

66
!!! note ""
77

8-
**✅ You can call ANY BigFunction from your Google Cloud Project** (*no install*).
8+
**✅ You can call ANY of the following public BigFunctions from your Google Cloud Project** (*no install*).
99

10+
- The functions are deployed in `bigfunctions` GCP project in 37 datasets for all of the 37 BigQuery regions.
11+
- They are public, so they can be called by anyone.
1012
- For any question or difficulties, please read [Getting Started](/bigfunctions/).
1113
- If you prefer to deploy the BigFunction in your own project, read [Getting Started](/bigfunctions/).
12-
13-
14+
- Found a bug? Please raise an issue [here](https://github.com/unytics/bigfunctions/issues/new/choose)
15+
16+
??? info "All BigFunctions Datasets >"
17+
18+
| Region | Dataset |
19+
|--------|---------|
20+
| `EU` | `bigfunctions.eu` |
21+
| `US` | `bigfunctions.us` |
22+
| `asia-east1` | `bigfunctions.asia_east1` |
23+
| `asia-east2` | `bigfunctions.asia_east2` |
24+
| `asia-northeast1` | `bigfunctions.asia_northeast1` |
25+
| `asia-northeast2` | `bigfunctions.asia_northeast2` |
26+
| `asia-northeast3` | `bigfunctions.asia_northeast3` |
27+
| `asia-south1` | `bigfunctions.asia_south1` |
28+
| `asia-south2` | `bigfunctions.asia_south2` |
29+
| `asia-southeast1` | `bigfunctions.asia_southeast1` |
30+
| `asia-southeast2` | `bigfunctions.asia_southeast2` |
31+
| `australia-southeast1` | `bigfunctions.australia_southeast1` |
32+
| `australia-southeast2` | `bigfunctions.australia_southeast2` |
33+
| `europe-central2` | `bigfunctions.europe_central2` |
34+
| `europe-north1` | `bigfunctions.europe_north1` |
35+
| `europe-southwest1` | `bigfunctions.europe_southwest1` |
36+
| `europe-west1` | `bigfunctions.europe_west1` |
37+
| `europe-west2` | `bigfunctions.europe_west2` |
38+
| `europe-west3` | `bigfunctions.europe_west3` |
39+
| `europe-west4` | `bigfunctions.europe_west4` |
40+
| `europe-west6` | `bigfunctions.europe_west6` |
41+
| `europe-west8` | `bigfunctions.europe_west8` |
42+
| `europe-west9` | `bigfunctions.europe_west9` |
43+
| `europe-west12` | `bigfunctions.europe_west12` |
44+
| `me-central1` | `bigfunctions.me_central1` |
45+
| `me-west1` | `bigfunctions.me_west1` |
46+
| `northamerica-northeast1` | `bigfunctions.northamerica_northeast1` |
47+
| `northamerica-northeast2` | `bigfunctions.northamerica_northeast2` |
48+
| `southamerica-east1` | `bigfunctions.southamerica_east1` |
49+
| `southamerica-west1` | `bigfunctions.southamerica_west1` |
50+
| `us-central1` | `bigfunctions.us_central1` |
51+
| `us-east1` | `bigfunctions.us_east1` |
52+
| `us-east4` | `bigfunctions.us_east4` |
53+
| `us-east5` | `bigfunctions.us_east5` |
54+
| `us-south1` | `bigfunctions.us_south1` |
55+
| `us-west1` | `bigfunctions.us_west1` |
56+
| `us-west2` | `bigfunctions.us_west2` |
57+
| `us-west3` | `bigfunctions.us_west3` |
58+
| `us-west4` | `bigfunctions.us_west4` |
1459

1560
## 📄 Overview
1661

‎bigfun/templates/data.md

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
---
2+
title: "Tables"
3+
description: "Catalog of open-source Tables"
4+
---
5+
6+
!!! note ""
7+
8+
**✅ The following public tables** are
9+
10+
- deployed in `bigfunctions` GCP project in 37 datasets for all of the 37 BigQuery regions.
11+
- They are public, so they can be read by anyone under the eventual license given in the description of the table.
12+
- For any question or difficulties, please read [Getting Started](/bigfunctions/).
13+
- If you prefer to create the tables in your own project, read [Getting Started](/bigfunctions/).
14+
- Found a bug? Please raise an issue [here](https://github.com/unytics/bigfunctions/issues/new/choose)
15+
16+
??? info "All BigFunctions Datasets >"
17+
18+
| Region | Dataset |
19+
|--------|---------|
20+
| `EU` | `bigfunctions.eu` |
21+
| `US` | `bigfunctions.us` |
22+
| `asia-east1` | `bigfunctions.asia_east1` |
23+
| `asia-east2` | `bigfunctions.asia_east2` |
24+
| `asia-northeast1` | `bigfunctions.asia_northeast1` |
25+
| `asia-northeast2` | `bigfunctions.asia_northeast2` |
26+
| `asia-northeast3` | `bigfunctions.asia_northeast3` |
27+
| `asia-south1` | `bigfunctions.asia_south1` |
28+
| `asia-south2` | `bigfunctions.asia_south2` |
29+
| `asia-southeast1` | `bigfunctions.asia_southeast1` |
30+
| `asia-southeast2` | `bigfunctions.asia_southeast2` |
31+
| `australia-southeast1` | `bigfunctions.australia_southeast1` |
32+
| `australia-southeast2` | `bigfunctions.australia_southeast2` |
33+
| `europe-central2` | `bigfunctions.europe_central2` |
34+
| `europe-north1` | `bigfunctions.europe_north1` |
35+
| `europe-southwest1` | `bigfunctions.europe_southwest1` |
36+
| `europe-west1` | `bigfunctions.europe_west1` |
37+
| `europe-west2` | `bigfunctions.europe_west2` |
38+
| `europe-west3` | `bigfunctions.europe_west3` |
39+
| `europe-west4` | `bigfunctions.europe_west4` |
40+
| `europe-west6` | `bigfunctions.europe_west6` |
41+
| `europe-west8` | `bigfunctions.europe_west8` |
42+
| `europe-west9` | `bigfunctions.europe_west9` |
43+
| `europe-west12` | `bigfunctions.europe_west12` |
44+
| `me-central1` | `bigfunctions.me_central1` |
45+
| `me-west1` | `bigfunctions.me_west1` |
46+
| `northamerica-northeast1` | `bigfunctions.northamerica_northeast1` |
47+
| `northamerica-northeast2` | `bigfunctions.northamerica_northeast2` |
48+
| `southamerica-east1` | `bigfunctions.southamerica_east1` |
49+
| `southamerica-west1` | `bigfunctions.southamerica_west1` |
50+
| `us-central1` | `bigfunctions.us_central1` |
51+
| `us-east1` | `bigfunctions.us_east1` |
52+
| `us-east4` | `bigfunctions.us_east4` |
53+
| `us-east5` | `bigfunctions.us_east5` |
54+
| `us-south1` | `bigfunctions.us_south1` |
55+
| `us-west1` | `bigfunctions.us_west1` |
56+
| `us-west2` | `bigfunctions.us_west2` |
57+
| `us-west3` | `bigfunctions.us_west3` |
58+
| `us-west4` | `bigfunctions.us_west4` |
59+
60+
61+
62+
63+
64+
65+
---
66+
67+
{% for table in tables %}
68+
69+
### {{ table.name }}
70+
<div style="position: relative; top: -2rem; margin-bottom: -2rem; text-align: right; z-index: 9999;">
71+
72+
<a href="{{ repo_url }}/blob/main/data/{{ table.name }}.yaml" title="Edit on GitHub" target="_blank"><svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 24 24"><path fill="#5d6cc0" d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"/></svg></a></div>
73+
74+
**Description**
75+
76+
{{ table.description }}
77+
78+
79+
**Schema**
80+
81+
| Column | Type | Description |
82+
|---------|------|--------------|
83+
{% for column in table.schema -%}
84+
| {{ column.name }} | {{ column.type }} | {{ column.description }} |
85+
{% endfor %}
86+
87+
---
88+
89+
{% endfor %}

0 commit comments

Comments
 (0)