Skip to content

Commit

Permalink
doc: generate doc for tables in data folder
Browse files Browse the repository at this point in the history
  • Loading branch information
unytics committed Nov 29, 2023
1 parent f25c486 commit ce14975
Show file tree
Hide file tree
Showing 11 changed files with 1,266 additions and 30 deletions.
45 changes: 36 additions & 9 deletions bigfun/generate_doc.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
TEMPLATE_MKDOCS_YAML = TEMPLATE_FOLDER + '/mkdocs.yml'
DOCUMENTATION_CONFIG_FILENAME = 'config_documentation.yaml'
BIGFUNCTIONS_FOLDER = 'bigfunctions'
TABLES_FOLDER = 'data'

if not os.path.isfile('README.md'):
print('INFO: CREATING A README.md FILE IN CURRENT DIRECTORY WHICH WILL BE THE ROOT CONTENT OF THE WEBSITE')
Expand All @@ -20,18 +21,31 @@


def get_bigfunctions():
bigfunctions = {}
bigfunctions = []
for filename in sorted(os.listdir(BIGFUNCTIONS_FOLDER)):
if not filename.endswith('.yaml'):
continue
conf = yaml.safe_load(open(f'bigfunctions/{filename}', encoding='utf-8').read())
conf = yaml.safe_load(open(f'{BIGFUNCTIONS_FOLDER}/{filename}', encoding='utf-8').read())
assert isinstance(conf, dict), f'Could not load yaml config of bigfunction `{filename}`'
name = filename.replace('.yaml', '')
conf['name'] = name
bigfunctions[name] = conf
bigfunctions.append(conf)
return bigfunctions


def get_tables():
tables = []
for filename in sorted(os.listdir(TABLES_FOLDER)):
if not filename.endswith('.yaml'):
continue
conf = yaml.safe_load(open(f'{TABLES_FOLDER}/{filename}', encoding='utf-8').read())
assert isinstance(conf, dict), f'Could not load yaml config of table `{filename}`'
name = filename.replace('.yaml', '')
conf['name'] = name
tables.append(conf)
return tables


# def get_documentation_config():
# if not os.path.isfile(DOCUMENTATION_CONFIG_FILENAME):
# print(f'WARNING: Could not find file `{DOCUMENTATION_CONFIG_FILENAME}` in current directory.')
Expand All @@ -43,6 +57,7 @@ def get_bigfunctions():


BIGFUNCTIONS = get_bigfunctions()
TABLES = get_tables()
# DOC_CONFIG = get_documentation_config()


Expand All @@ -52,35 +67,47 @@ def get_bigfunctions():

class BasePage:

name = ''

def generate(self):
context = self.get_context()
template = f'{TEMPLATE_FOLDER}/{self.template_filename}'
template = f'{TEMPLATE_FOLDER}/{self.name}.md'
content = jinja2.Template(open(template, encoding='utf-8').read()).render(**context)
with open(f'bigfunctions/README.md', 'w', encoding='utf-8') as out:
with open(f'{self.name}/README.md', 'w', encoding='utf-8') as out:
out.write(content)

def get_context(self):
raise NotImplementedError()


class ReferencePage(BasePage):
class BigFunctionsPage(BasePage):

template_filename = 'bigfunctions.md'
name = 'bigfunctions'

def get_context(self):
categories = CONF['bigfunctions_categories']
for category in categories:
category['bigfunctions'] = [bigfunction for bigfunction in BIGFUNCTIONS.values() if bigfunction['category'] == category['name']]
category['bigfunctions'] = [bigfunction for bigfunction in BIGFUNCTIONS if bigfunction['category'] == category['name']]
return {
'datasets': CONF['bigfunctions_datasets'],
'repo_url': CONF['repo_url'],
'categories': categories,
}

class TablesPage(BasePage):

name = 'data'

def get_context(self):
return {
'tables': TABLES,
'repo_url': CONF['repo_url'],
}


def generate_doc():
ReferencePage().generate()
BigFunctionsPage().generate()
TablesPage().generate()


if __name__ == '__main__':
Expand Down
51 changes: 48 additions & 3 deletions bigfun/templates/bigfunctions.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,57 @@ description: "Catalog of open-source BigFunctions"

!!! note ""

**✅ You can call ANY BigFunction from your Google Cloud Project** (*no install*).
**✅ You can call ANY of the following public BigFunctions from your Google Cloud Project** (*no install*).

- The functions are deployed in `bigfunctions` GCP project in 37 datasets for all of the 37 BigQuery regions.
- They are public, so they can be called by anyone.
- For any question or difficulties, please read [Getting Started](/bigfunctions/).
- If you prefer to deploy the BigFunction in your own project, read [Getting Started](/bigfunctions/).


- Found a bug? Please raise an issue [here](https://github.com/unytics/bigfunctions/issues/new/choose)

??? info "All BigFunctions Datasets >"

| Region | Dataset |
|--------|---------|
| `EU` | `bigfunctions.eu` |
| `US` | `bigfunctions.us` |
| `asia-east1` | `bigfunctions.asia_east1` |
| `asia-east2` | `bigfunctions.asia_east2` |
| `asia-northeast1` | `bigfunctions.asia_northeast1` |
| `asia-northeast2` | `bigfunctions.asia_northeast2` |
| `asia-northeast3` | `bigfunctions.asia_northeast3` |
| `asia-south1` | `bigfunctions.asia_south1` |
| `asia-south2` | `bigfunctions.asia_south2` |
| `asia-southeast1` | `bigfunctions.asia_southeast1` |
| `asia-southeast2` | `bigfunctions.asia_southeast2` |
| `australia-southeast1` | `bigfunctions.australia_southeast1` |
| `australia-southeast2` | `bigfunctions.australia_southeast2` |
| `europe-central2` | `bigfunctions.europe_central2` |
| `europe-north1` | `bigfunctions.europe_north1` |
| `europe-southwest1` | `bigfunctions.europe_southwest1` |
| `europe-west1` | `bigfunctions.europe_west1` |
| `europe-west2` | `bigfunctions.europe_west2` |
| `europe-west3` | `bigfunctions.europe_west3` |
| `europe-west4` | `bigfunctions.europe_west4` |
| `europe-west6` | `bigfunctions.europe_west6` |
| `europe-west8` | `bigfunctions.europe_west8` |
| `europe-west9` | `bigfunctions.europe_west9` |
| `europe-west12` | `bigfunctions.europe_west12` |
| `me-central1` | `bigfunctions.me_central1` |
| `me-west1` | `bigfunctions.me_west1` |
| `northamerica-northeast1` | `bigfunctions.northamerica_northeast1` |
| `northamerica-northeast2` | `bigfunctions.northamerica_northeast2` |
| `southamerica-east1` | `bigfunctions.southamerica_east1` |
| `southamerica-west1` | `bigfunctions.southamerica_west1` |
| `us-central1` | `bigfunctions.us_central1` |
| `us-east1` | `bigfunctions.us_east1` |
| `us-east4` | `bigfunctions.us_east4` |
| `us-east5` | `bigfunctions.us_east5` |
| `us-south1` | `bigfunctions.us_south1` |
| `us-west1` | `bigfunctions.us_west1` |
| `us-west2` | `bigfunctions.us_west2` |
| `us-west3` | `bigfunctions.us_west3` |
| `us-west4` | `bigfunctions.us_west4` |

## 📄 Overview

Expand Down
89 changes: 89 additions & 0 deletions bigfun/templates/data.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
---
title: "Tables"
description: "Catalog of open-source Tables"
---

!!! note ""

**✅ The following public tables** are

- deployed in `bigfunctions` GCP project in 37 datasets for all of the 37 BigQuery regions.
- They are public, so they can be read by anyone under the eventual license given in the description of the table.
- For any question or difficulties, please read [Getting Started](/bigfunctions/).
- If you prefer to create the tables in your own project, read [Getting Started](/bigfunctions/).
- Found a bug? Please raise an issue [here](https://github.com/unytics/bigfunctions/issues/new/choose)

??? info "All BigFunctions Datasets >"

| Region | Dataset |
|--------|---------|
| `EU` | `bigfunctions.eu` |
| `US` | `bigfunctions.us` |
| `asia-east1` | `bigfunctions.asia_east1` |
| `asia-east2` | `bigfunctions.asia_east2` |
| `asia-northeast1` | `bigfunctions.asia_northeast1` |
| `asia-northeast2` | `bigfunctions.asia_northeast2` |
| `asia-northeast3` | `bigfunctions.asia_northeast3` |
| `asia-south1` | `bigfunctions.asia_south1` |
| `asia-south2` | `bigfunctions.asia_south2` |
| `asia-southeast1` | `bigfunctions.asia_southeast1` |
| `asia-southeast2` | `bigfunctions.asia_southeast2` |
| `australia-southeast1` | `bigfunctions.australia_southeast1` |
| `australia-southeast2` | `bigfunctions.australia_southeast2` |
| `europe-central2` | `bigfunctions.europe_central2` |
| `europe-north1` | `bigfunctions.europe_north1` |
| `europe-southwest1` | `bigfunctions.europe_southwest1` |
| `europe-west1` | `bigfunctions.europe_west1` |
| `europe-west2` | `bigfunctions.europe_west2` |
| `europe-west3` | `bigfunctions.europe_west3` |
| `europe-west4` | `bigfunctions.europe_west4` |
| `europe-west6` | `bigfunctions.europe_west6` |
| `europe-west8` | `bigfunctions.europe_west8` |
| `europe-west9` | `bigfunctions.europe_west9` |
| `europe-west12` | `bigfunctions.europe_west12` |
| `me-central1` | `bigfunctions.me_central1` |
| `me-west1` | `bigfunctions.me_west1` |
| `northamerica-northeast1` | `bigfunctions.northamerica_northeast1` |
| `northamerica-northeast2` | `bigfunctions.northamerica_northeast2` |
| `southamerica-east1` | `bigfunctions.southamerica_east1` |
| `southamerica-west1` | `bigfunctions.southamerica_west1` |
| `us-central1` | `bigfunctions.us_central1` |
| `us-east1` | `bigfunctions.us_east1` |
| `us-east4` | `bigfunctions.us_east4` |
| `us-east5` | `bigfunctions.us_east5` |
| `us-south1` | `bigfunctions.us_south1` |
| `us-west1` | `bigfunctions.us_west1` |
| `us-west2` | `bigfunctions.us_west2` |
| `us-west3` | `bigfunctions.us_west3` |
| `us-west4` | `bigfunctions.us_west4` |






---

{% for table in tables %}

### {{ table.name }}
<div style="position: relative; top: -2rem; margin-bottom: -2rem; text-align: right; z-index: 9999;">

<a href="{{ repo_url }}/blob/main/data/{{ table.name }}.yaml" title="Edit on GitHub" target="_blank"><svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 24 24"><path fill="#5d6cc0" d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"/></svg></a></div>

**Description**

{{ table.description }}


**Schema**

| Column | Type | Description |
|---------|------|--------------|
{% for column in table.schema -%}
| {{ column.name }} | {{ column.type }} | {{ column.description }} |
{% endfor %}

---

{% endfor %}
Loading

0 comments on commit ce14975

Please sign in to comment.