Skip to content

Commit

Permalink
Merge pull request #878 from stadtnavi/chore/update-parse-layers-sheet
Browse files Browse the repository at this point in the history 
refactor(parse-layers-sheet): read layers csv from url
  • Loading branch information
@andreashelms
andreashelms authored Jan 13, 2025
2 parents fd3da60 + 7a06232 commit d0bbff3
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 108 deletions.
5 changes: 2 additions & 3 deletions docs/Common-stadtnavi-commands.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,10 @@ This watches if any of the submodules has been changed and compiles it on-the-fl
Map layers config from layers spreadsheet can be converted from CSV to JSON running

```
node ./scripts/parse-layers-sheet.js
node ./scripts/parse-layers-sheet.js "https://docs.google.com/spreadsheets/d/{key}/gviz/tq?tqx=out:csv&sheet={sheet}"
```

Before running the script make sure to have CSV export of layers spreadsheet
in place and to have set the correct input and output file names in the script.
Before running make sure to have have set the correct output file name in the script.

Created JSON file can be used to replace files in

Expand Down
233 changes: 128 additions & 105 deletions scripts/parse-layers-sheet.js
View file
Original file line number Diff line number Diff line change
@@ -1,119 +1,142 @@
/* eslint-disable dot-notation */
const fs = require('fs');
const axios = require('axios');
const csv = require('csv-parser');
const { Readable } = require('stream');
const fs = require('fs');

const results = [];

fs.createReadStream('layers.csv')
.pipe(csv())
.on('data', data => results.push(data))
.on('end', () => {
const mergedLayers = {};
const url = process.argv[2];
if (!url) {
// eslint-disable-next-line no-console
console.error('Please provide a URL as a command-line argument.');
process.exit(1);
}

axios
.get(url)
.then(response => {
const csvStream = new Readable();
csvStream.push(response.data);
csvStream.push(null);

csvStream
.pipe(csv())
.on('data', data => results.push(data))
.on('end', () => {
const mergedLayers = {};

results.forEach(row => {
if (!mergedLayers[row['Oberkategorie Code']]) {
mergedLayers[row['Oberkategorie Code']] = {
code: row['Oberkategorie Code'],
translations: {
de: row['Oberkategorie Deutsch'],
en: row['Oberkategorie Englisch'],
},
categories: [],
};
}
results.forEach(row => {
if (!mergedLayers[row['Oberkategorie Code']]) {
mergedLayers[row['Oberkategorie Code']] = {
code: row['Oberkategorie Code'],
translations: {
de: row['Oberkategorie Deutsch'],
en: row['Oberkategorie Englisch'],
},
categories: [],
};
}

let category = mergedLayers[row['Oberkategorie Code']].categories.find(
cat => cat.code === row['Kategorie Code'],
);
let category = mergedLayers[
row['Oberkategorie Code']
].categories.find(cat => cat.code === row['Kategorie Code']);

if (!category) {
category = {
code: row['Kategorie Code'],
translations: {
de: row['Kategorie Deutsch'],
en: row['Kategorie Englisch'],
},
categories: [],
properties: null,
};
mergedLayers[row['Oberkategorie Code']].categories.push(category);
}
if (!category) {
category = {
code: row['Kategorie Code'],
translations: {
de: row['Kategorie Deutsch'],
en: row['Kategorie Englisch'],
},
categories: [],
properties: null,
};
mergedLayers[row['Oberkategorie Code']].categories.push(category);
}

if (row['Unterkategorie Code']) {
category.categories.push({
code: row['Unterkategorie Code'],
translations: {
de: row['Unterkategorie Deutsch'],
en: row['Unterkategorie Englisch'],
},
properties: {
layer: {
type: row['Layer-Typ'],
url: row['Url'],
priority: parseInt(row['Priorisierung bei Überlappung'], 10),
min_zoom: parseInt(row['Minimales Zoomlevel'], 10),
},
icon: {
svg: row['Icon .svg'],
origin: row['Herkunft Symbol'],
background_color: row['Icon: Farbe Hintergrund_Code'],
color: row['Icon: Farbe Symbol'],
},
attributes: row['Eigenschaften']
.split(',')
.map(attr => attr.trim()),
osm: {
filter: row['OSM-Filter'],
example: row['Beispielobjekt aus OSM'],
wiki: row['OSM-Wiki-Seite'],
overpass_query: row['Overpass Query'],
},
misc: {
comment: row['Kommentar'],
origin: row['Herkunft'],
},
},
if (row['Unterkategorie Code']) {
category.categories.push({
code: row['Unterkategorie Code'],
translations: {
de: row['Unterkategorie Deutsch'],
en: row['Unterkategorie Englisch'],
},
properties: {
layer: {
type: row['Layer-Typ'],
url: row['Url'],
priority: parseInt(row['Priorisierung bei Überlappung'], 10),
min_zoom: parseInt(row['Minimales Zoomlevel'], 10),
},
icon: {
svg: row['Icon .svg'],
origin: row['Herkunft Symbol'],
background_color: row['Icon: Farbe Hintergrund_Code'],
color: row['Icon: Farbe Symbol'],
},
attributes: row['Eigenschaften']
.split(',')
.map(attr => attr.trim()),
osm: {
filter: row['OSM-Filter'],
example: row['Beispielobjekt aus OSM'],
wiki: row['OSM-Wiki-Seite'],
overpass_query: row['Overpass Query'],
},
misc: {
comment: row['Kommentar'],
origin: row['Herkunft'],
},
},
});
} else {
category.properties = {
layer: {
type: row['Layer-Typ'],
url: row['Url'],
priority: parseInt(row['Priorisierung bei Überlappung'], 10),
min_zoom: parseInt(row['Minimales Zoomlevel'], 10),
},
icon: {
svg: row['Icon .svg'],
origin: row['Herkunft Symbol'],
background_color: row['Icon: Farbe Hintergrund_Code'],
color: row['Icon: Farbe Symbol'],
},
attributes: row['Eigenschaften']
.split(',')
.map(attr => attr.trim()),
osm: {
filter: row['OSM-Filter'],
example: row['Beispielobjekt aus OSM'],
wiki: row['OSM-Wiki-Seite'],
overpass_query: row['Overpass Query'],
},
misc: {
comment: row['Kommentar'],
origin: row['Herkunft'],
},
};
}
});
} else {
category.properties = {
layer: {
type: row['Layer-Typ'],
url: row['Url'],
priority: parseInt(row['Priorisierung bei Überlappung'], 10),
min_zoom: parseInt(row['Minimales Zoomlevel'], 10),
},
icon: {
svg: row['Icon .svg'],
origin: row['Herkunft Symbol'],
background_color: row['Icon: Farbe Hintergrund_Code'],
color: row['Icon: Farbe Symbol'],
},
attributes: row['Eigenschaften'].split(',').map(attr => attr.trim()),
osm: {
filter: row['OSM-Filter'],
example: row['Beispielobjekt aus OSM'],
wiki: row['OSM-Wiki-Seite'],
overpass_query: row['Overpass Query'],
},
misc: {
comment: row['Kommentar'],
origin: row['Herkunft'],
},
};
}
});

// Remove categories property if it's an empty array
Object.values(mergedLayers).forEach(layer => {
layer.categories.forEach(category => {
if (category.categories.length === 0) {
// eslint-disable-next-line no-param-reassign
delete category.categories;
}
});
});
// Remove categories property if it's an empty array
Object.values(mergedLayers).forEach(layer => {
layer.categories.forEach(category => {
if (category.categories.length === 0) {
// eslint-disable-next-line no-param-reassign
delete category.categories;
}
});
});

const jsonOutput = Object.values(mergedLayers);
const jsonOutput = Object.values(mergedLayers);

fs.writeFileSync('layers.json', JSON.stringify(jsonOutput, null, 2));
fs.writeFileSync('layers.json', JSON.stringify(jsonOutput, null, 2));
});
})
.catch(error => {
// eslint-disable-next-line no-console
console.error('Error fetching the CSV file:', error.message);
});

0 comments on commit d0bbff3

Please sign in to comment.