Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(parse-layers-sheet): read layers csv from url #878

Merged
merged 2 commits into from
Jan 13, 2025
Merged

refactor(parse-layers-sheet): read layers csv from url #878

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b20375b
refactor(parse-layers-sheet): read layers csv from url
Jan 9, 2025
7a06232
chore(parse-layers-sheet): update docs
Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions docs/Common-stadtnavi-commands.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,10 @@ This watches if any of the submodules has been changed and compiles it on-the-fl
Map layers config from layers spreadsheet can be converted from CSV to JSON running

```
node ./scripts/parse-layers-sheet.js
node ./scripts/parse-layers-sheet.js "https://docs.google.com/spreadsheets/d/{key}/gviz/tq?tqx=out:csv&sheet={sheet}"
```

Before running the script make sure to have CSV export of layers spreadsheet
in place and to have set the correct input and output file names in the script.
Before running make sure to have have set the correct output file name in the script.

Created JSON file can be used to replace files in

Expand Down
233 changes: 128 additions & 105 deletions scripts/parse-layers-sheet.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,119 +1,142 @@
/* eslint-disable dot-notation */
const fs = require('fs');
const axios = require('axios');
const csv = require('csv-parser');
const { Readable } = require('stream');
const fs = require('fs');

const results = [];

fs.createReadStream('layers.csv')
.pipe(csv())
.on('data', data => results.push(data))
.on('end', () => {
const mergedLayers = {};
const url = process.argv[2];
if (!url) {
// eslint-disable-next-line no-console
console.error('Please provide a URL as a command-line argument.');
process.exit(1);
}

axios
.get(url)
.then(response => {
const csvStream = new Readable();
csvStream.push(response.data);
csvStream.push(null);

csvStream
.pipe(csv())
.on('data', data => results.push(data))
.on('end', () => {
const mergedLayers = {};

results.forEach(row => {
if (!mergedLayers[row['Oberkategorie Code']]) {
mergedLayers[row['Oberkategorie Code']] = {
code: row['Oberkategorie Code'],
translations: {
de: row['Oberkategorie Deutsch'],
en: row['Oberkategorie Englisch'],
},
categories: [],
};
}
results.forEach(row => {
if (!mergedLayers[row['Oberkategorie Code']]) {
mergedLayers[row['Oberkategorie Code']] = {
code: row['Oberkategorie Code'],
translations: {
de: row['Oberkategorie Deutsch'],
en: row['Oberkategorie Englisch'],
},
categories: [],
};
}

let category = mergedLayers[row['Oberkategorie Code']].categories.find(
cat => cat.code === row['Kategorie Code'],
);
let category = mergedLayers[
row['Oberkategorie Code']
].categories.find(cat => cat.code === row['Kategorie Code']);

if (!category) {
category = {
code: row['Kategorie Code'],
translations: {
de: row['Kategorie Deutsch'],
en: row['Kategorie Englisch'],
},
categories: [],
properties: null,
};
mergedLayers[row['Oberkategorie Code']].categories.push(category);
}
if (!category) {
category = {
code: row['Kategorie Code'],
translations: {
de: row['Kategorie Deutsch'],
en: row['Kategorie Englisch'],
},
categories: [],
properties: null,
};
mergedLayers[row['Oberkategorie Code']].categories.push(category);
}

if (row['Unterkategorie Code']) {
category.categories.push({
code: row['Unterkategorie Code'],
translations: {
de: row['Unterkategorie Deutsch'],
en: row['Unterkategorie Englisch'],
},
properties: {
layer: {
type: row['Layer-Typ'],
url: row['Url'],
priority: parseInt(row['Priorisierung bei Überlappung'], 10),
min_zoom: parseInt(row['Minimales Zoomlevel'], 10),
},
icon: {
svg: row['Icon .svg'],
origin: row['Herkunft Symbol'],
background_color: row['Icon: Farbe Hintergrund_Code'],
color: row['Icon: Farbe Symbol'],
},
attributes: row['Eigenschaften']
.split(',')
.map(attr => attr.trim()),
osm: {
filter: row['OSM-Filter'],
example: row['Beispielobjekt aus OSM'],
wiki: row['OSM-Wiki-Seite'],
overpass_query: row['Overpass Query'],
},
misc: {
comment: row['Kommentar'],
origin: row['Herkunft'],
},
},
if (row['Unterkategorie Code']) {
category.categories.push({
code: row['Unterkategorie Code'],
translations: {
de: row['Unterkategorie Deutsch'],
en: row['Unterkategorie Englisch'],
},
properties: {
layer: {
type: row['Layer-Typ'],
url: row['Url'],
priority: parseInt(row['Priorisierung bei Überlappung'], 10),
min_zoom: parseInt(row['Minimales Zoomlevel'], 10),
},
icon: {
svg: row['Icon .svg'],
origin: row['Herkunft Symbol'],
background_color: row['Icon: Farbe Hintergrund_Code'],
color: row['Icon: Farbe Symbol'],
},
attributes: row['Eigenschaften']
.split(',')
.map(attr => attr.trim()),
osm: {
filter: row['OSM-Filter'],
example: row['Beispielobjekt aus OSM'],
wiki: row['OSM-Wiki-Seite'],
overpass_query: row['Overpass Query'],
},
misc: {
comment: row['Kommentar'],
origin: row['Herkunft'],
},
},
});
} else {
category.properties = {
layer: {
type: row['Layer-Typ'],
url: row['Url'],
priority: parseInt(row['Priorisierung bei Überlappung'], 10),
min_zoom: parseInt(row['Minimales Zoomlevel'], 10),
},
icon: {
svg: row['Icon .svg'],
origin: row['Herkunft Symbol'],
background_color: row['Icon: Farbe Hintergrund_Code'],
color: row['Icon: Farbe Symbol'],
},
attributes: row['Eigenschaften']
.split(',')
.map(attr => attr.trim()),
osm: {
filter: row['OSM-Filter'],
example: row['Beispielobjekt aus OSM'],
wiki: row['OSM-Wiki-Seite'],
overpass_query: row['Overpass Query'],
},
misc: {
comment: row['Kommentar'],
origin: row['Herkunft'],
},
};
}
});
} else {
category.properties = {
layer: {
type: row['Layer-Typ'],
url: row['Url'],
priority: parseInt(row['Priorisierung bei Überlappung'], 10),
min_zoom: parseInt(row['Minimales Zoomlevel'], 10),
},
icon: {
svg: row['Icon .svg'],
origin: row['Herkunft Symbol'],
background_color: row['Icon: Farbe Hintergrund_Code'],
color: row['Icon: Farbe Symbol'],
},
attributes: row['Eigenschaften'].split(',').map(attr => attr.trim()),
osm: {
filter: row['OSM-Filter'],
example: row['Beispielobjekt aus OSM'],
wiki: row['OSM-Wiki-Seite'],
overpass_query: row['Overpass Query'],
},
misc: {
comment: row['Kommentar'],
origin: row['Herkunft'],
},
};
}
});

// Remove categories property if it's an empty array
Object.values(mergedLayers).forEach(layer => {
layer.categories.forEach(category => {
if (category.categories.length === 0) {
// eslint-disable-next-line no-param-reassign
delete category.categories;
}
});
});
// Remove categories property if it's an empty array
Object.values(mergedLayers).forEach(layer => {
layer.categories.forEach(category => {
if (category.categories.length === 0) {
// eslint-disable-next-line no-param-reassign
delete category.categories;
}
});
});

const jsonOutput = Object.values(mergedLayers);
const jsonOutput = Object.values(mergedLayers);

fs.writeFileSync('layers.json', JSON.stringify(jsonOutput, null, 2));
fs.writeFileSync('layers.json', JSON.stringify(jsonOutput, null, 2));
});
})
.catch(error => {
// eslint-disable-next-line no-console
console.error('Error fetching the CSV file:', error.message);
});
Loading