Merge branch 'dev'

README.md

@@ -6,10 +6,10 @@ Mod portal link: https://mods.factorio.com/mod/L0laapk3_FactorioMaps
 
 # How to Install
 **Note that this program now only runs on 64 bit python version 3.6 or higher.**
-1. Download FactorioMaps to `%appdata%\mods\`, either from the [mod portal](https://mods.factorio.com/mod/L0laapk3_FactorioMaps) (The mod does not need to be enabled to work) and then unzipping it, or from [the github releases page](https://github.com/L0laapk3/FactorioMaps/releases). 
-1. Install the latest version of [**64 bit** python 3](https://www.python.org/downloads/). (Do *not* install python 2.) 
+1. Download FactorioMaps to `%appdata%\mods\`, either from the [mod portal](https://mods.factorio.com/mod/L0laapk3_FactorioMaps) (The mod does not need to be enabled to work) and then unzipping it, or from [the github releases page](https://github.com/L0laapk3/FactorioMaps/releases).
+1. Install the latest version of [**64 bit** python 3](https://www.python.org/downloads/). (Do *not* install python 2.)
 Make sure to do a select the "add python to PATH" and "install pip" options.
-1. Inside the factoriomaps folder, install the required pip packages: `python -m pip install -r packages.txt`.
+1. Inside the factoriomaps folder, install the required pip packages: `python -m pip install -r requirements.txt`.
 
 # How to Use
 1. Make sure you close factorio before starting the process.
@@ -61,11 +61,12 @@ Heres a list of flags that `auto.py` can accept:
 | `--delete` | Deletes the output folder specified before running the script. |
 | `--dry` | Skips starting factorio, making screenshots and doing the main steps, only execute setting up and finishing of script. |
 | `--force-lib-update` | Forces an update of the web dependencies. |
-
+| `--temp-dir` | Use a custom temporary directory. |
+
 Image quality settings can be changed in the top of `zoom.py`.
 
 # Result folder estimates
-You can expect the resulting folders to take up approx. (very rough estimate) 15 times the savefile size per timestamp per daytime for day images and 10 times for night images. The intermediate total disk usage will be much higher, 10 times the final result or more. If this is a problem for you, go put a +1 on [#46](https://github.com/L0laapk3/FactorioMaps/issues/46). 
+You can expect the resulting folders to take up approx. (very rough estimate) 15 times the savefile size per timestamp per daytime for day images and 10 times for night images. The intermediate total disk usage will be much higher, 10 times the final result or more. If this is a problem for you, go put a +1 on [#46](https://github.com/L0laapk3/FactorioMaps/issues/46).
 Of course the processing time depends very heavely on your system specs, but a rough estimate is an hour per timestamp per daytime per 50 MB of savefile.
 
 # Hosting this on a server

auto.py

@@ -11,18 +11,19 @@
 from pathlib import Path
 
 try:
- with Path(__file__, "..", "packages.txt").resolve().open("r", encoding="utf-8") as f:
+ with Path(__file__, "..", "requirements.txt").resolve().open("r", encoding="utf-8") as f:
  pkg_resources.require(f.read().splitlines())
 except (DistributionNotFound, VersionConflict) as ex:
  traceback.print_exc()
- print("\nDependencies not met. Run `pip install -r packages.txt` to install missing dependencies.")
+ print("\nDependencies not met. Run `pip install -r requirements.txt` to install missing dependencies.")
  sys.exit(1)
 
 import glob
 import argparse
 import configparser
 import datetime
 import json
+import errno
 import math
 import multiprocessing as mp
 import random
@@ -55,9 +56,9 @@
 
 userFolder = Path(__file__, "..", "..", "..").resolve()
 
-def naturalSort(l): 
- convert = lambda text: int(text) if text.isdigit() else text.lower() 
- alphanum_key = lambda key: [ convert(c) for c in re.split('(\d+)', key) ] 
+def naturalSort(l):
+ convert = lambda text: int(text) if text.isdigit() else text.lower()
+ alphanum_key = lambda key: [ convert(c) for c in re.split('(\d+)', key) ]
  return sorted(l, key = alphanum_key)
 
 def printErase(arg):
@@ -79,7 +80,10 @@ def startGameAndReadGameLogs(results, condition, exeWithArgs, isSteam, tmpDir, p
  prevPrinted = False
  def handleGameLine(line, isFirst):
  if isFirst and not re.match(r'^ *\d+\.\d{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d; Factorio (\d+\.\d+\.\d+) \(build (\d+), [^)]+\)$', line):
- raise Exception("Unrecognised output from factorio (maybe your version is outdated or too new?)\n\nOutput from factorio:\n" + line)
+ suggestion = "maybe your version is outdated or too new?"
+ if line.endswith('Error Util.cpp:83: weakly_canonical: Incorrect function.'):
+ suggestion = "maybe your temp directory is on a ramdisk?"
+ raise RuntimeError(f"Unrecognised output from factorio ({suggestion})\n\nOutput from factorio:\n{line}")
 
  nonlocal prevPrinted
  line = line.rstrip('\n')
@@ -89,7 +93,7 @@ def handleGameLine(line, isFirst):
  return
 
  prevPrinted = False
- 
+
  m = re.match(r'^\ *\d+(?:\.\d+)? *Script *@__L0laapk3_FactorioMaps__\/data-final-fixes\.lua:\d+: FactorioMaps_Output_RawTagPaths:([^:]+):(.*)$', line, re.IGNORECASE)
  if m is not None:
  rawTags[m.group(1)] = m.group(2)
@@ -114,10 +118,10 @@ def handleGameLine(line, isFirst):
 
 
  with os.fdopen(pipeOut, 'r') as pipef:
- 
+
  if isSteam:
- printErase("using steam launch hack.")
- 
+ printErase("using steam launch hack")
+
  attrs = ('pid', 'name', 'create_time')
 
  # on some devices, the previous check wasn't enough apparently, so explicitely wait until the log file is created.
@@ -404,6 +408,7 @@ def kill(pid, onlyStall=False):
  parser.add_argument("targetname", nargs="?", help="output folder name for the generated snapshots.")
  parser.add_argument("savename", nargs="*", help="Names of the savegames to generate snapshots from. If no savegames are provided the latest save or the save matching outfolder will be gerated. Glob patterns are supported.")
  parser.add_argument("--force-lib-update", action="store_true", help="Forces an update of the web dependencies.")
+ parser.add_argument('--temp-dir', '--tempdir', type=lambda p: Path(p).resolve(), help='Set a custom temporary directory to use (this is only needed if the defualt one is on a RAM disk, which Factorio does not support).')
 
  args = parser.parse_args()
  if args.verbose > 0:
@@ -419,7 +424,7 @@ def kill(pid, onlyStall=False):
  timestamp, filePath = max(
  (save.stat().st_mtime, save)
  for save in saves.iterdir()
- if save.stem not in {"_autosave1", "_autosave2", "_autosave3"}
+ if not save.stem.startswith("_autosave") and save.name != "steam_autocloud.vdf"
  )
  foldername = filePath.stem
  print("No save name passed. Using most recent save: %s" % foldername)
@@ -434,10 +439,10 @@ def kill(pid, onlyStall=False):
 
  if not globResults:
  print(f'Cannot find savefile: "{saveName}"')
- raise ValueError(f'Cannot find savefile: "{saveName}"')
+ raise IOError(f"savefile {saveName!r} not found in {str(saves)!r}")
  results = [save for save in globResults if save.is_file()]
  for result in results:
- saveGames.add(result.stem)
+ saveGames.add(result.relative_to(saves).as_posix())
 
  saveGames = naturalSort(list(saveGames))
 
@@ -459,15 +464,20 @@ def kill(pid, onlyStall=False):
  "Program Files (x86)/Steam/steamapps/common/Factorio/bin/x64/factorio.exe",
  "Steam/steamapps/common/Factorio/bin/x64/factorio.exe",
  ]
+ def driveExists(drive):
+ try:
+ return Path(f"{drive}:/").exists()
+ except (OSError, PermissionError):
+ return False
  availableDrives = [
- "%s:/" % d for d in string.ascii_uppercase if Path(f"{d}:/").exists()
+ "%s:/" % d for d in string.ascii_uppercase if driveExists(d)
  ]
  possibleFactorioPaths = unixPaths
  if args.steam == 0:
  possibleFactorioPaths += [ drive + path for drive in availableDrives for path in windowsPathsStandalone ]
  if args.standalone == 0:
  possibleFactorioPaths += [ drive + path for drive in availableDrives for path in windowsPathsSteam ]
- 
+
  try:
  factorioPath = next(
  x
@@ -546,7 +556,12 @@ def kill(pid, onlyStall=False):
  buildAutorun(args, workfolder, foldername, isFirstSnapshot, setDaytime)
  isFirstSnapshot = False
 
- with TemporaryDirectory(prefix="FactorioMaps-") as tmpDir:
+ if args.temp_dir is not None:
+ try:
+ os.makedirs(args.temp_dir)
+ except OSError:
+ pass
+ with TemporaryDirectory(prefix="FactorioMaps-", dir=args.temp_dir) as tmpDir:
  configPath = buildConfig(args, tmpDir, args.basepath)
 
  pid = None
@@ -556,7 +571,7 @@ def kill(pid, onlyStall=False):
 
  launchArgs = [
  '--load-game',
- str(Path(userFolder, 'saves', savename).absolute()),
+ str(Path(userFolder, 'saves', *(savename.split('/'))).absolute()),
  '--disable-audio',
  '--config',
  str(configPath),
@@ -575,19 +590,19 @@ def kill(pid, onlyStall=False):
  # try to find steam
  try:
  from winreg import OpenKey, HKEY_CURRENT_USER, ConnectRegistry, QueryValueEx, REG_SZ
- 
+
  key = OpenKey(ConnectRegistry(None, HKEY_CURRENT_USER), r'Software\Valve\Steam')
  val, valType = QueryValueEx(key, 'SteamExe')
  if valType != REG_SZ:
- raise FileNotFoundError( errno.ENOENT, os.strerror(errno.ENOENT), "SteamExe")
+ raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), "SteamExe")
  steamPath = Path(val)
  except (ImportError, FileNotFoundError) as e:
  # fallback to old method
  if os.name == "nt":
  steamPath = Path(factorioPath, "..", "..", "..", "..", "..", "..", "steam.exe")
  else:
  steamPath = Path(factorioPath, "..", "..", "..", "..", "..", "..", "steam")
- 
+
  if steamPath and steamPath.exists(): # found a steam executable
  usedSteamLaunchHack = True
  exeWithArgs = [
@@ -738,6 +753,14 @@ def refZoom():
  os.remove(os.path.join(workfolder, "mapInfo.out.json"))
 
 
+ # List of length 3 tuples:
+ # mod name in lowercase (e.g. `krastorio2`, `fnei`)
+ # (major version string, minor version string, patch version string, bool if the mod's a zipfile)
+ # mod full ID in original casing (e.g. `Krastorio2_1.1.4`, `FNEI_0.4.1`)
+ #
+ # Does not include mods that don't have versions in
+ # their names, such as mods manually installed from
+ # source.
  modVersions = sorted(
  map(lambda m: (m.group(2).lower(), (m.group(3), m.group(4), m.group(5), m.group(6) is None), m.group(1)),
  filter(lambda m: m,
@@ -800,13 +823,14 @@ def addTag(tags, itemType, itemName, force=False):
  if not mod[1][3]: #true if mod is zip
  zipPath = os.path.join(args.basepath, args.mod_path, mod[2] + ".zip")
  with ZipFile(zipPath, 'r') as zipObj:
+ internalFolder = os.path.commonpath(zipObj.namelist())
  if len(icons) == 1:
- zipInfo = zipObj.getinfo(os.path.join(mod[2], icon + ".png").replace('\\', '/'))
+ zipInfo = zipObj.getinfo(os.path.join(internalFolder, icon + ".png").replace('\\', '/'))
  zipInfo.filename = os.path.basename(dest)
  zipObj.extract(zipInfo, os.path.dirname(os.path.realpath(dest)))
  src = None
  else:
- src = zipObj.extract(os.path.join(mod[2], icon + ".png").replace('\\', '/'), os.path.join(tempfile.gettempdir(), "FactorioMaps"))
+ src = zipObj.extract(os.path.join(internalFolder, icon + ".png").replace('\\', '/'), os.path.join(tempfile.gettempdir(), "FactorioMaps"))
  else:
  src = os.path.join(args.basepath, args.mod_path, mod[2], icon + ".png")
 

data-final-fixes.lua

@@ -11,7 +11,11 @@ require("json")
 
 
 
-
+local function typeCheck(item, itemType, field, expectedType, fieldName)
+ if type(field) ~= expectedType then
+ error("\n\n\n\n\n\nItem malformed: '" .. tostring(item.name) .. "' (item type: " .. itemType .. ")\n\nentity." .. fieldName .. " = " .. tostring(field) .. " (" .. type(field) .. " != " .. expectedType .. ")\n\n\nPlease report this error with the mod the item originates from.\n\n\n\n\n")
+ end
+end
 
 
 local function index(entity, type)
@@ -24,12 +28,20 @@ local function index(entity, type)
  -- }
 
 
+ typeCheck(entity, type, entity.name, "string", "name")
  local path = ""
  if entity.icon ~= nil then
+ typeCheck(entity, type, entity.icon, "string", "icon")
  path = entity.icon:sub(1, -5)
  else
  for i, icon in pairs(entity.icons) do
+ typeCheck(entity, type, icon.icon, "string", "icons[" .. i .. "].icon")
  if icon.tint ~= nil then
+ typeCheck(entity, type, icon.tint, "table", "icons[" .. i .. "].tint")
+ if icon.tint["r"] ~= nil then typeCheck(entity, type, icon.tint["r"], "number", "icons[" .. i .. "].tint.r") end
+ if icon.tint["g"] ~= nil then typeCheck(entity, type, icon.tint["g"], "number", "icons[" .. i .. "].tint.g") end
+ if icon.tint["b"] ~= nil then typeCheck(entity, type, icon.tint["b"], "number", "icons[" .. i .. "].tint.b") end
+ if icon.tint["a"] ~= nil then typeCheck(entity, type, icon.tint["a"], "number", "icons[" .. i .. "].tint.a") end
  path = path .. "|" .. icon.icon:sub(1, -5) .. "?" ..
  math.floor((icon.tint["r"] or 0)*255+0.5) .. "%" ..
  math.floor((icon.tint["g"] or 0)*255+0.5) .. "%" ..

info.json

@@ -1,6 +1,6 @@
 {
  "name": "L0laapk3_FactorioMaps",
- "version": "4.2.0",
+ "version": "4.3.0",
  "title": "FactorioMaps",
  "author": "L0laapk3",
  "contact": "https://github.com/L0laapk3/",

updates.json

@@ -74,5 +74,12 @@
  "Remove orderedset dependency",
  "Add partial support for rich text in item labels",
  "Security fix that prevents html injection"
+ ],
+ "4.3.0": [
+ "Allow for custom temporary directory for usage with RAM-disks by Huntfx",
+ "Support savegames located in subdirectories by sonowz",
+ "Fix OSError/PermissionError in availableDrives by sonowz",
+ "Ignore steam_autocloud file",
+ "Implemented malformed item checking"
  ]
 }

web/index.js

@@ -42,7 +42,7 @@ for (let i = 0; i < mapInfo.maps.length; i++) {
  for (const surface of Object.keys(map.surfaces)) {
  let layer = map.surfaces[surface];
 
- 
+
  if (!layer.captured)
  continue;
 
@@ -71,7 +71,7 @@ for (let i = 0; i < mapInfo.maps.length; i++) {
 
  console.assert(row.length % 3 == 0); //corrupted data, prevent infinite loop
  let j = 3, y = B64Parse(0) - 2**17;
- 
+
  if (!globalTileNightIndex[surface][layer.zoom.max][y]){
  globalTileNightIndex[surface][layer.zoom.max][y] = {};
  globalTileIndex[surface][layer.zoom.max][y] = {};
@@ -127,7 +127,7 @@ for (let i = 0; i < mapInfo.maps.length; i++) {
  layersByTimestamp[i][surface] = {};
  map.surfaces[surface].layers = {};
 
- 
+
  layer.tags.sort((a, b) => a.position.y - b.position.y);
  const mapInfoTimeLayer = Object.values(mapInfo.maps).find(m => m.path == map.path);
  for (const tag of layer.tags) {
@@ -185,7 +185,7 @@ for (let i = 0; i < mapInfo.maps.length; i++) {
  });
  }
  marker.link = link;
- 
+
  if (subMarkers) {
  subMarkers.push(marker);
  } else {
@@ -320,7 +320,7 @@ function updateLabels() {
  }
  }
  break;
- 
+
  }
  } else if (!shouldBeVisible && label.visible)
  for (const marker of [label.marker, ...label.subMarkers || []])
@@ -442,7 +442,7 @@ try {
  timestamp = split[6];
  if (!isNaN(parseInt(split[7])))
  timestamp += "-" + split[7];
- } 
+ }
  }
 } catch (_) {
  window.location.href = "#";
@@ -550,13 +550,13 @@ if (layersByTimestamp.length > 1 && true) {
  let timeLabels = layersByTimestamp.map(function(layer, i) {
  return {
  name: mapInfo.maps[i].path + "h",
- position: max == min ? i / (layersByTimestamp.length - 1) : i * 30/sliderHeight + (parseInt(mapInfo.maps[i].path) - min) / (max - min) * (1 - (layersByTimestamp.length - 1) * 30/sliderHeight),
+ position: max == min || layersByTimestamp.length * 30/sliderHeight > 1 ? i / (layersByTimestamp.length - 1) : i * 30/sliderHeight + (parseInt(mapInfo.maps[i].path) - min) / (max - min) * (1 - (layersByTimestamp.length - 1) * 30/sliderHeight),
  layers: Object.values(layer).map(s => ["day", "night"].map(n => s[n]).filter(l => l)).flat()
  }
  });
 
 
- 
+
  let initialTime;
  for (let i = 0; i < timeLabels.length; i++) {
  if (parseFloat(timestamp) < parseInt(timeLabels[i].name)) {