Merge pull request #6 from invertase/feat/unpublished-command-filter

Author: Salakar

.gitignore

@@ -8,3 +8,4 @@
 /pubspec.lock
 /flutterfire/
 /workspaces
+.DS_Store

CHANGELOG.md

@@ -0,0 +1,85 @@
+## 0.2.0
+
+- Added a new filter for filtering published or unpublished packages: `--[no-]published`.
+  - Unpublished in this case means the package either does not exist on the Pub registry or the current local version of the package is not yet published to the Pub registry.
+- Added a new command to pretty print currently unpublished packages: `melos unpublished`.
+
+#### Example `--[no-]published` usage
+
+Example logging out all unpublished packages and their versions:
+
+```bash
+mike@MikeMacMini fe_ff_master % melos exec --no-published --ignore="*example*" -- echo MELOS_PACKAGE_NAME MELOS_PACKAGE_VERSION
+$ melos exec --no-published
+   └> echo MELOS_PACKAGE_NAME MELOS_PACKAGE_VERSION
+       └> RUNNING (in 12 packages)
+
+[firebase_admob]: firebase_admob 0.9.3+4
+[firebase_analytics_platform_interface]: firebase_analytics_platform_interface 1.0.3
+[firebase_auth]: firebase_auth 0.17.0-dev.1
+[firebase_auth_web]: firebase_auth_web 0.2.0-dev.1
+[firebase_core]: firebase_core 0.5.0-dev.2
+[firebase_crashlytics]: firebase_crashlytics 0.1.4+1
+[firebase_database]: firebase_database 4.0.0-dev.1
+[firebase_dynamic_links]: firebase_dynamic_links 0.5.3
+[firebase_ml_vision]: firebase_ml_vision 0.9.5
+[firebase_remote_config]: firebase_remote_config 0.3.1+1
+[firebase_storage]: firebase_storage 4.0.0-dev.1
+
+$ melos exec --no-published
+   └> echo MELOS_PACKAGE_NAME MELOS_PACKAGE_VERSION
+       └> SUCCESS
+mike@MikeMacMini fe_ff_master %
+```
+
+#### Example `unpublished` usage
+
+```bash
+mike@MikeMacMini fe_ff_master % melos unpublished --ignore="*example*"
+$ melos unpublished
+   └> /Users/mike/Documents/Projects/Flutter/fe_ff_master
+
+Reading registry for package information... SUCCESS
+
+$ melos unpublished
+   └> /Users/mike/Documents/Projects/Flutter/fe_ff_master
+       └> UNPUBLISHED PACKAGES (12 packages)
+           └> firebase_analytics_platform_interface
+               • Local:   1.0.3
+               • Remote:  1.0.1
+           └> cloud_functions
+               • Local:   0.6.0-dev.2
+               • Remote:  0.6.0-dev.1
+           └> firebase_core
+               • Local:   0.5.0-dev.2
+               • Remote:  0.5.0-dev.1
+           └> firebase_auth_web
+               • Local:   0.2.0-dev.1
+               • Remote:  0.1.3+1
+           └> firebase_dynamic_links
+               • Local:   0.5.3
+               • Remote:  0.5.1
+           └> firebase_crashlytics
+               • Local:   0.1.4+1
+               • Remote:  0.1.3+3
+           └> firebase_admob
+               • Local:   0.9.3+4
+               • Remote:  0.9.3+2
+           └> firebase_ml_vision
+               • Local:   0.9.5
+               • Remote:  0.9.4
+           └> firebase_remote_config
+               • Local:   0.3.1+1
+               • Remote:  0.3.1
+           └> firebase_database
+               • Local:   4.0.0-dev.1
+               • Remote:  3.1.6
+           └> firebase_auth
+               • Local:   0.17.0-dev.1
+               • Remote:  0.9.0
+           └> firebase_storage
+               • Local:   4.0.0-dev.1
+               • Remote:  3.1.6
+
+mike@MikeMacMini fe_ff_master %
+```

lib/src/command/unpublished.dart

@@ -0,0 +1,75 @@
+import 'dart:io';
+
+import 'package:args/command_runner.dart' show Command;
+import 'package:pool/pool.dart' show Pool;
+
+import '../common/logger.dart';
+import '../common/package.dart';
+import '../common/workspace.dart';
+
+class UnpublishedCommand extends Command {
+  @override
+  final String name = 'unpublished';
+
+  @override
+  final List<String> aliases = ['unp'];
+
+  @override
+  final String description =
+      'Discover and list unpublished packages or package versions in your repository.';
+
+  @override
+  void run() async {
+    logger.stdout(
+        '${logger.ansi.yellow}\$${logger.ansi.noColor} ${logger.ansi.emphasized("melos unpublished")}');
+    logger.stdout(
+        '   └> ${logger.ansi.cyan}${logger.ansi.emphasized(currentWorkspace.path)}${logger.ansi.noColor}\n');
+    var readRegistryProgress =
+        logger.progress('Reading registry for package information');
+
+    var pool = Pool(10);
+    var unpublishedPackages = <MelosPackage>[];
+    var latestPackageVersion = <String, String>{};
+    await pool.forEach<MelosPackage, void>(currentWorkspace.packages,
+        (package) {
+      return package.getPublishedVersions().then((versions) async {
+        if (versions.isEmpty || !versions.contains(package.version)) {
+          unpublishedPackages.add(package);
+          if (versions.isEmpty) {
+            latestPackageVersion[package.name] = 'none';
+          } else {
+            latestPackageVersion[package.name] = versions[0];
+          }
+        }
+      });
+    }).drain();
+
+    readRegistryProgress.finish(
+        message: '${logger.ansi.green}SUCCESS${logger.ansi.noColor}',
+        showTiming: true);
+
+    logger.stdout('');
+    logger.stdout(
+        '${logger.ansi.yellow}\$${logger.ansi.noColor} ${logger.ansi.emphasized("melos unpublished")}');
+    logger.stdout(
+        '   └> ${logger.ansi.cyan}${logger.ansi.emphasized(currentWorkspace.path)}${logger.ansi.noColor}');
+    if (unpublishedPackages.isNotEmpty) {
+      logger.stdout(
+          '       └> ${logger.ansi.red}${logger.ansi.emphasized('UNPUBLISHED PACKAGES')}${logger.ansi.noColor} (${unpublishedPackages.length} packages)');
+      unpublishedPackages.forEach((package) {
+        logger.stdout(
+            '           └> ${logger.ansi.yellow}${package.name}${logger.ansi.noColor}');
+        logger.stdout(
+            '               ${logger.ansi.bullet} ${logger.ansi.green}Local:${logger.ansi.noColor}   ${package.version ?? 'none'}');
+        logger.stdout(
+            '               ${logger.ansi.bullet} ${logger.ansi.cyan}Remote:${logger.ansi.noColor}  ${latestPackageVersion[package.name]}');
+      });
+      logger.stdout('');
+      exit(1);
+    } else {
+      logger.stdout(
+          '       └> ${logger.ansi.green}${logger.ansi.emphasized('NO UNPUBLISHED PACKAGES')}${logger.ansi.noColor}');
+      logger.stdout('');
+    }
+  }
+}

lib/src/command_runner.dart

@@ -3,6 +3,7 @@ import 'dart:io';
 import 'package:args/args.dart';
 import 'package:args/command_runner.dart';
 import 'package:cli_util/cli_logging.dart';
+import 'package:melos/src/command/unpublished.dart';
 
 import 'command/bootstrap.dart';
 import 'command/clean.dart';
@@ -27,6 +28,12 @@ class MelosCommandRunner extends CommandRunner {
         help:
             'Exclude private packages (`publish_to: none`). They are included by default.');
 
+    argParser.addFlag('published',
+        negatable: true,
+        defaultsTo: null,
+        help:
+            'Filter packages where the current local package version exists on pub.dev. Or "-no-published" to filter packages that have not had their current version published yet.');
+
     argParser.addMultiOption('scope',
         help: 'Include only packages with names matching the given glob.');
 
@@ -45,6 +52,7 @@ class MelosCommandRunner extends CommandRunner {
     addCommand(BootstrapCommand());
     addCommand(CleanCommand());
     addCommand(RunCommand());
+    addCommand(UnpublishedCommand());
   }
 
   @override
@@ -67,6 +75,7 @@ class MelosCommandRunner extends CommandRunner {
     await currentWorkspace.loadPackages(
       scope: argResults['scope'] as List<String>,
       skipPrivate: argResults['no-private'] as bool,
+      published: argResults['published'] as bool,
       ignore: argResults['ignore'] as List<String>,
       dirExists: argResults['dir-exists'] as List<String>,
       fileExists: argResults['file-exists'] as List<String>,

lib/src/common/package.dart

@@ -1,7 +1,9 @@
 import 'dart:async';
+import 'dart:convert';
 import 'dart:io';
 
 import 'package:yaml/yaml.dart';
+import 'package:http/http.dart' as http;
 
 import '../pub/pub_file.dart';
 import '../pub/pub_file_flutter_dependencies.dart';
@@ -33,6 +35,7 @@ const String kWeb = 'web';
 
 class MelosPackage {
   final Map _yamlContents;
+  List<String> _registryVersions;
 
   final String _name;
 
@@ -134,7 +137,8 @@ class MelosPackage {
           '$exampleParentPackagePath${Platform.pathSeparator}pubspec.yaml'));
       if (exampleParentPackage != null) {
         environment['MELOS_PARENT_PACKAGE_NAME'] = exampleParentPackage.name;
-        environment['MELOS_PARENT_PACKAGE_VERSION'] = exampleParentPackage.version;
+        environment['MELOS_PARENT_PACKAGE_VERSION'] =
+            exampleParentPackage.version;
         environment['MELOS_PARENT_PACKAGE_PATH'] = exampleParentPackage.path;
       }
     }
@@ -160,6 +164,28 @@ class MelosPackage {
     });
   }
 
+  Future<List<String>> getPublishedVersions() async {
+    if (_registryVersions != null) {
+      return _registryVersions;
+    }
+    var url = 'https://pub.dev/packages/$name.json';
+    var response = await http.get(url);
+    if (response.statusCode == 404) {
+      return [];
+    } else if (response.statusCode != 200) {
+      throw Exception(
+          'Error reading pub.dev registry for package "$name" (HTTP Status ${response.statusCode}), response: ${response.body}');
+    }
+    var versions = <String>[];
+    var versionsRaw = json.decode(response.body)['versions'] as List<dynamic>;
+    versionsRaw.forEach((element) {
+      versions.add(element as String);
+    });
+    versions.sort();
+    _registryVersions = versions.reversed.toList();
+    return _registryVersions;
+  }
+
   void clean() {
     PackagesPubFile.fromDirectory(path).delete();
     FlutterPluginsPubFile.fromDirectory(path).delete();
@@ -227,4 +253,9 @@ class MelosPackage {
     if (_yamlContents['publish_to'].runtimeType != String) return false;
     return _yamlContents['publish_to'] == 'none';
   }
+
+  @override
+  String toString() {
+    return 'MelosPackage[$name@$version]';
+  }
 }

lib/src/common/workspace.dart

@@ -5,6 +5,7 @@ import 'package:args/args.dart';
 import 'package:glob/glob.dart';
 import 'package:meta/meta.dart';
 import 'package:path/path.dart';
+import 'package:pool/pool.dart';
 import 'package:yamlicious/yamlicious.dart';
 
 import '../pub/pub_deps_list.dart';
@@ -57,7 +58,8 @@ class MelosWorkspace {
       List<String> ignore,
       List<String> dirExists,
       List<String> fileExists,
-      bool skipPrivate}) async {
+      bool skipPrivate,
+      bool published}) async {
     if (_packages != null) return Future.value(_packages);
 
     final packageGlobs = _config.packages;
@@ -133,13 +135,33 @@ class MelosWorkspace {
     }
 
     if (skipPrivate) {
-        // Whether we should skip packages with 'publish_to: none' set.
+      // Whether we should skip packages with 'publish_to: none' set.
       filterResult = filterResult.where((package) {
         return !package.isPrivate();
       });
     }
 
-    _packages = await filterResult.toList();
+    if (published != null) {
+      _packages = await filterResult.toList();
+      // Pooling to parrellize registry requests for performance.
+      var pool = Pool(10);
+      var packagesFilteredWithPublishStatus = <MelosPackage>[];
+      await pool.forEach<MelosPackage, void>(_packages, (package) {
+        return package.getPublishedVersions().then((versions) async {
+          var isOnPubRegistry = versions.contains(package.version);
+          if (published == false && !isOnPubRegistry) {
+            return packagesFilteredWithPublishStatus.add(package);
+          }
+          if (published == true && isOnPubRegistry) {
+            return packagesFilteredWithPublishStatus.add(package);
+          }
+        });
+      }).drain();
+      _packages = packagesFilteredWithPublishStatus;
+    } else {
+      _packages = await filterResult.toList();
+    }
+
     _packages.sort((a, b) {
       return a.name.compareTo(b.name);
     });

lib/src/pub/pub_file_flutter_plugins.dart

@@ -1,7 +1,6 @@
 import 'dart:io';
 
 import '../common/package.dart';
-import '../common/utils.dart' as utils;
 import '../common/workspace.dart';
 import '../pub/pub_file.dart';
 

pubspec.yaml

@@ -1,11 +1,12 @@
 name: "melos"
 description: "A tool for managing Dart projects with multiple packages. Inspired by JavaScripts Lerna package."
-version: "0.1.0-13.0.pre"
+version: "0.2.0"
 homepage: "https://github.com/invertase/melos"
 executables:
   melos:
 dependencies:
   args: ^1.6.0
+  http: ^0.12.2
   pool: ^1.4.0
   collection: ^1.14.12
   string_scanner: ^1.0.5