C#: Restore projects and collect dependencies for projects in the same folder sequentially.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs

@@ -16,6 +16,11 @@ internal class Assets
  {
  private readonly ILogger logger;
 
+ /// <summary>
+ /// Contains the dependencies found in the parsed asset files.
+ /// </summary>
+ public DependencyContainer Dependencies { get; } = new();
+
  internal Assets(ILogger logger)
  {
  this.logger = logger;
@@ -72,7 +77,7 @@ private record class ReferenceInfo(string? Type, Dictionary<string, object>? Com
  /// "json.net"
  /// }
  /// </summary>
- private void AddPackageDependencies(JObject json, DependencyContainer dependencies)
+ private void AddPackageDependencies(JObject json)
  {
  // If there is more than one framework we need to pick just one.
  // To ensure stability we pick one based on the lexicographic order of
@@ -107,13 +112,13 @@ private void AddPackageDependencies(JObject json, DependencyContainer dependenci
  // If this is a framework reference then include everything.
  if (FrameworkPackageNames.AllFrameworks.Any(framework => name.StartsWith(framework)))
  {
- dependencies.AddFramework(name);
+ Dependencies.AddFramework(name);
  }
  return;
  }
 
  info.Compile
- .ForEach(r => dependencies.Add(name, r.Key));
+ .ForEach(r => Dependencies.Add(name, r.Key));
  });
 
  return;
@@ -149,7 +154,7 @@ private void AddPackageDependencies(JObject json, DependencyContainer dependenci
  /// "microsoft.netcore.app.ref"
  /// }
  /// </summary>
- private void AddFrameworkDependencies(JObject json, DependencyContainer dependencies)
+ private void AddFrameworkDependencies(JObject json)
  {
 
  var frameworks = json
@@ -178,21 +183,21 @@ private void AddFrameworkDependencies(JObject json, DependencyContainer dependen
 
  references
  .Properties()
- .ForEach(f => dependencies.AddFramework($"{f.Name}.Ref".ToLowerInvariant()));
+ .ForEach(f => Dependencies.AddFramework($"{f.Name}.Ref".ToLowerInvariant()));
  }
 
  /// <summary>
  /// Parse `json` as project.assets.json content and add relative paths to the dependencies
  /// (together with used package information) required for compilation.
  /// </summary>
  /// <returns>True if parsing succeeds, otherwise false.</returns>
- public bool TryParse(string json, DependencyContainer dependencies)
+ public bool TryParse(string json)
  {
  try
  {
  var obj = JObject.Parse(json);
- AddPackageDependencies(obj, dependencies);
- AddFrameworkDependencies(obj, dependencies);
+ AddPackageDependencies(obj);
+ AddFrameworkDependencies(obj);
  return true;
  }
  catch (Exception e)
@@ -217,19 +222,24 @@ private static bool TryReadAllText(string path, ILogger logger, [NotNullWhen(ret
  }
  }
 
- public static DependencyContainer GetCompilationDependencies(ILogger logger, IEnumerable<string> assets)
+ /// <summary>
+ /// Add the dependencies from the assets file to the dependencies.
+ /// </summary>
+ /// <param name="asset">Path to an asset file.</param>
+ public void AddDependencies(string asset)
  {
- var parser = new Assets(logger);
- var dependencies = new DependencyContainer();
- assets.ForEach(asset =>
+ if (TryReadAllText(asset, logger, out var json))
  {
- if (TryReadAllText(asset, logger, out var json))
- {
- parser.TryParse(json, dependencies);
- }
- });
- return dependencies;
+ TryParse(json);
+ }
  }
+
+ /// <summary>
+ /// Add the dependencies from the assets files to the dependencies.
+ /// </summary>
+ /// <param name="assets">Collection of paths to asset files.</param>
+ public void AddDependenciesRange(IEnumerable<string> assets) =>
+ assets.ForEach(AddDependencies);
  }
 
  internal static class JsonExtensions

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyContainer.cs

@@ -12,7 +12,7 @@ internal class DependencyContainer
  /// <summary>
  /// Paths to dependencies required for compilation.
  /// </summary>
- public List<string> Paths { get; } = new();
+ public HashSet<string> Paths { get; } = new();
 
  /// <summary>
  /// Packages that are used as a part of the required dependencies.
@@ -68,4 +68,18 @@ public void AddFramework(string framework)
  Packages.Add(GetPackageName(p));
  }
  }
-}
+
+ internal static class DependencyContainerExtensions
+ {
+ /// <summary>
+ /// Flatten a list of containers into a single container.
+ /// </summary>
+ public static DependencyContainer Flatten(this IEnumerable<DependencyContainer> container) =>
+ container.Aggregate(new DependencyContainer(), (acc, c) =>
+ {
+ acc.Paths.UnionWith(c.Paths);
+ acc.Packages.UnionWith(c.Packages);
+ return acc;
+ });
+ }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -3,8 +3,6 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Security.Cryptography;
-using System.Text;
 using System.Threading.Tasks;
 
 using Semmle.Util;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -144,11 +144,12 @@ public HashSet<AssemblyLookupLocation> Restore()
  logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
  }
 
- var restoredProjects = RestoreSolutions(out var assets1);
+ var restoredProjects = RestoreSolutions(out var container);
  var projects = fileProvider.Projects.Except(restoredProjects);
- RestoreProjects(projects, out var assets2);
+ RestoreProjects(projects, out var containers);
 
- var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
+ containers.Add(container);
+ var dependencies = containers.Flatten();
 
  var paths = dependencies
  .Paths
@@ -198,14 +199,14 @@ private List<string> GetReachableFallbackNugetFeeds()
  /// As opposed to RestoreProjects this is not run in parallel using PLINQ
  /// as `dotnet restore` on a solution already uses multiple threads for restoring
  /// the projects (this can be disabled with the `--disable-parallel` flag).
- /// Populates assets with the relative paths to the assets files generated by the restore.
+ /// Populates dependencies with the relevant dependencies from the assets files generated by the restore.
  /// Returns a list of projects that are up to date with respect to restore.
  /// </summary>
- private IEnumerable<string> RestoreSolutions(out IEnumerable<string> assets)
+ private IEnumerable<string> RestoreSolutions(out DependencyContainer dependencies)
  {
  var successCount = 0;
  var nugetSourceFailures = 0;
- var assetFiles = new List<string>();
+ var assets = new Assets(logger);
  var projects = fileProvider.Solutions.SelectMany(solution =>
  {
  logger.LogInfo($"Restoring solution {solution}...");
@@ -218,10 +219,10 @@ private IEnumerable<string> RestoreSolutions(out IEnumerable<string> assets)
  {
  nugetSourceFailures++;
  }
- assetFiles.AddRange(res.AssetsFilePaths);
+ assets.AddDependenciesRange(res.AssetsFilePaths);
  return res.RestoredProjects;
  }).ToList();
- assets = assetFiles;
+ dependencies = assets.Dependencies;
  compilationInfoContainer.CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
  compilationInfoContainer.CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
  compilationInfoContainer.CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
@@ -231,33 +232,42 @@ private IEnumerable<string> RestoreSolutions(out IEnumerable<string> assets)
  /// <summary>
  /// Executes `dotnet restore` on all projects in projects.
  /// This is done in parallel for performance reasons.
- /// Populates assets with the relative paths to the assets files generated by the restore.
+ /// Populates dependencies with the relative paths to the assets files generated by the restore.
  /// </summary>
  /// <param name="projects">A list of paths to project files.</param>
- private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
+ private void RestoreProjects(IEnumerable<string> projects, out List<DependencyContainer> dependencies)
  {
  var successCount = 0;
  var nugetSourceFailures = 0;
- var assetFiles = new List<string>();
+ List<DependencyContainer> collectedDependencies = [];
  var sync = new object();
- Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = DependencyManager.Threads }, project =>
+ var projectGroups = projects.GroupBy(Path.GetDirectoryName);
+ Parallel.ForEach(projectGroups, new ParallelOptions { MaxDegreeOfParallelism = DependencyManager.Threads }, projectGroup =>
  {
- logger.LogInfo($"Restoring project {project}...");
- var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
- lock (sync)
+ var assets = new Assets(logger);
+ foreach (var project in projectGroup)
  {
- if (res.Success)
- {
- successCount++;
- }
- if (res.HasNugetPackageSourceError)
+ logger.LogInfo($"Restoring project {project}...");
+ var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+ assets.AddDependenciesRange(res.AssetsFilePaths);
+ lock (sync)
  {
- nugetSourceFailures++;
+ if (res.Success)
+ {
+ successCount++;
+ }
+ if (res.HasNugetPackageSourceError)
+ {
+ nugetSourceFailures++;
+ }
  }
- assetFiles.AddRange(res.AssetsFilePaths);
+ }
+ lock (sync)
+ {
+ collectedDependencies.Add(assets.Dependencies);
  }
  });
- assets = assetFiles;
+ dependencies = collectedDependencies;
  compilationInfoContainer.CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
  compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
  }