refactor for Cid-serde integration

Signed-off-by: jorgee <[email protected]>

Author: jorgee

modules/nextflow/src/main/groovy/nextflow/cli/CmdCid.groovy

@@ -44,7 +44,6 @@ class CmdCid extends CmdBase implements UsageAware {
         void show(ConfigMap config, List<String> args)
         void lineage(ConfigMap config, List<String> args)
         void diff(ConfigMap config, List<String> args)
-        void query(ConfigMap config, List<String> args)
     }
 
     interface SubCmd {
@@ -65,7 +64,6 @@ class CmdCid extends CmdBase implements UsageAware {
         commands << new CmdShow()
         commands << new CmdLineage()
         commands << new CmdDiff()
-        commands << new CmdQuery()
     }
 
     @Parameter(hidden = true)
@@ -260,30 +258,4 @@ class CmdCid extends CmdBase implements UsageAware {
 
     }
 
-    class CmdQuery implements SubCmd {
-
-        @Override
-        String getName() { 'query' }
-
-        @Override
-        String getDescription() {
-            return 'Search data in descriptions'
-        }
-
-        void apply(List<String> args) {
-            if (args.size() != 1) {
-                println("ERROR: Incorrect number of parameters")
-                usage()
-                return
-            }
-            operation.query(config, args)
-        }
-
-        @Override
-        void usage() {
-            println description
-            println "Usage: nextflow $NAME $name <URIQuery>"
-        }
-
-    }
 }

modules/nextflow/src/test/groovy/nextflow/cli/CmdCidTest.groovy

@@ -17,6 +17,8 @@
 
 package nextflow.cli
 
+import nextflow.data.cid.serde.CidEncoder
+
 import java.nio.file.Files
 
 import nextflow.SysEnv
@@ -28,7 +30,6 @@ import nextflow.data.cid.model.Parameter
 import nextflow.data.cid.model.TaskOutput
 import nextflow.data.cid.model.TaskRun
 import nextflow.data.cid.model.WorkflowOutput
-import nextflow.data.cid.serde.CidEncoder
 import nextflow.plugin.Plugins
 import org.junit.Rule
 import spock.lang.Specification
@@ -177,7 +178,7 @@ class CmdCidTest extends Specification {
 
         then:
             stdout.size() == 1
-            stdout[0] == "No entry found for cid://12345."
+            stdout[0] == "No entries found for cid://12345."
 
         cleanup:
             folder?.deleteDir()
@@ -258,4 +259,38 @@ class CmdCidTest extends Specification {
 
     }
 
+    def 'should show query results'(){
+        given:
+        def folder = Files.createTempDirectory('test').toAbsolutePath()
+        def configFile = folder.resolve('nextflow.config')
+        configFile.text = "workflow.data.enabled = true\nworkflow.data.store.location = '$folder'".toString()
+        def cidFile = folder.resolve(".meta/12345/.data.json")
+        Files.createDirectories(cidFile.parent)
+        def launcher = Mock(Launcher){
+            getOptions() >> new CliOptions(config: [configFile.toString()])
+        }
+        def encoder = new CidEncoder().withPrettyPrint(true)
+        def entry = new WorkflowOutput("path/to/file",new Checksum("45372qe","nextflow","standard"),
+                "cid://123987/file.bam", 1234, 123456789, 123456789, null)
+        def jsonSer = encoder.encode(entry)
+        def expectedOutput = jsonSer
+        cidFile.text = jsonSer
+        when:
+        def cidCmd = new CmdCid(launcher: launcher, args: ["show", "cid:///?type=WorkflowOutput"])
+        cidCmd.run()
+        def stdout = capture
+                .toString()
+                .readLines()// remove the log part
+                .findResults { line -> !line.contains('DEBUG') ? line : null }
+                .findResults { line -> !line.contains('INFO') ? line : null }
+                .findResults { line -> !line.contains('plugin') ? line : null }
+
+        then:
+        stdout.size() == expectedOutput.readLines().size()
+        stdout.join('\n') == expectedOutput
+
+        cleanup:
+        folder?.deleteDir()
+    }
+
 }

modules/nextflow/src/test/groovy/nextflow/script/OutputDslTest.groovy

@@ -77,9 +77,9 @@ class OutputDslTest extends Specification {
             "file2","${outputDir}/barbar/file2.txt"
             """.stripIndent()
         and:
-        1 * session.notifyFilePublish(outputDir.resolve('foo/file1.txt'), file1)
-        1 * session.notifyFilePublish(outputDir.resolve('barbar/file2.txt'), file2)
-        1 * session.notifyFilePublish(outputDir.resolve('index.csv'))
+        1 * session.notifyFilePublish(outputDir.resolve('foo/file1.txt'), file1, null)
+        1 * session.notifyFilePublish(outputDir.resolve('barbar/file2.txt'), file2, null)
+        1 * session.notifyFilePublish(outputDir.resolve('index.csv'), null, null)
 
         cleanup:
         SysEnv.pop()

modules/nf-cid/src/main/nextflow/data/cid/CidStore.groovy

@@ -53,6 +53,11 @@ interface CidStore extends Closeable {
      */
     CidHistoryLog getHistoryLog()
 
-    List<Object> query(URI queryString)
+    /**
+     * Search for cid entries.
+     * @queryString Json-path like query string. (Only simple and nested field operators are supported(No array, wildcards,etc.)
+     * @return List of Cid object's fulfilling the queryString
+     */
+    List<CidSerializable> search(String queryString)
 
 }

modules/nf-cid/src/main/nextflow/data/cid/CidUtils.groovy

@@ -0,0 +1,110 @@
+package nextflow.data.cid
+
+import groovy.util.logging.Slf4j
+import nextflow.data.cid.fs.CidPath
+import nextflow.data.cid.serde.CidSerializable
+
+import java.nio.file.Path
+
+@Slf4j
+class CidUtils {
+
+    public static List query(CidStore store, URI uri) {
+        String key = uri.authority ? uri.authority + uri.path : uri.path
+        try {
+            if (key == CidPath.SEPARATOR) {
+                return store.search(uri.query)
+            } else {
+                return searchPath(store, key, uri.query ? parseQuery(uri.query) : null)
+            }
+        } catch(Throwable e){
+            log.debug("Exception querying $uri. $e.message")
+            return []
+        }
+
+    }
+
+    private static List<Object> searchPath(CidStore store, String path, Map<String, String> params, String[] childs = []) {
+        final results = new LinkedList<Object>()
+        final object = store.load(path)
+        if (object) {
+            if (childs && childs.size() > 0) {
+                final output = navigate(object, childs.join('.'))
+                if (output) {
+                    treatObject(output, params, results)
+                } else {
+                    throw new FileNotFoundException("Cid object $path/${childs ? childs.join('/') : ''} not found.")
+                }
+            } else {
+                treatObject(object, params, results)
+            }
+        } else {
+            // If there isn't metadata check the parent to check if it is a subfolder of a task/workflow output
+            final currentPath = Path.of(path)
+            final parent = currentPath.getParent()
+            if (parent) {
+                ArrayList<String> newChilds = new ArrayList<String>()
+                newChilds.add(currentPath.getFileName().toString())
+                newChilds.addAll(childs)
+                return searchPath(store, parent.toString(), params, newChilds as String[])
+            } else {
+                throw new FileNotFoundException("Cid object $path/${childs ? childs.join('/') : ''} not found.")
+            }
+        }
+        return results
+    }
+
+    protected static void treatObject(def object, Map<String, String> params, List<Object> results) {
+        if (params) {
+            if (object instanceof Collection) {
+                (object as Collection).forEach { treatObject(it, params, results) }
+            } else if (checkParams(object, params)) {
+                results.add(object)
+            }
+        } else {
+            results.add(object)
+        }
+    }
+
+    public static Map<String, String> parseQuery(String queryString) {
+        return queryString.split('&').collectEntries {
+            it.split('=').collect { URLDecoder.decode(it, 'UTF-8') }
+        } as Map<String, String>
+    }
+
+    public static boolean checkParams(Object object, Map<String,String> params) {
+        for (final entry : params.entrySet()) {
+            final value = navigate(object, entry.key)
+            if (!value || value.toString() != entry.value.toString() ) {
+                return false
+            }
+        }
+        return true
+    }
+
+    protected static Object navigate(Object obj, String path){
+        if (!obj)
+            return null
+        try{
+            // type has been replaced by class when evaluating CidSerializable objects
+            if (obj instanceof CidSerializable && path == 'type')
+                return obj.getClass()?.simpleName
+            path.tokenize('.').inject(obj) { current, key ->
+                if (current == null) return null
+
+                if (current instanceof Map) {
+                    return current[key] // Navigate Map properties
+                }
+
+                if (current.metaClass.hasProperty(current, key)) {
+                    return current.getAt(key) // Navigate Object properties
+                }
+                log.debug("No property found for $key")
+                return null // Property not found
+            }
+        } catch (Throwable e) {
+            log.debug("Error navigating to $path in object", e)
+            return null
+        }
+    }
+}

modules/nf-cid/src/main/nextflow/data/cid/DefaultCidStore.groovy

@@ -17,7 +17,6 @@
 
 package nextflow.data.cid
 
-import groovy.json.JsonSlurper
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 
@@ -27,7 +26,6 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.attribute.BasicFileAttributes
 
-import nextflow.data.cid.fs.CidPath
 import nextflow.data.cid.serde.CidEncoder
 import nextflow.data.cid.serde.CidSerializable
 import nextflow.data.config.DataConfig
@@ -84,7 +82,7 @@ class DefaultCidStore implements CidStore {
         final path = metaLocation.resolve("$key/$METADATA_FILE")
         log.debug("Loading from path $path")
         if (path.exists())
-            return encoder.decode(path.text)
+            return encoder.decode(path.text) as CidSerializable
         log.debug("File for key $key not found")
         return null
     }
@@ -107,26 +105,18 @@ class DefaultCidStore implements CidStore {
     void close() throws IOException { }
 
     @Override
-    List<Object> query(URI uri) {
-        log.debug("Query $uri.query, Path: $uri.path, Scheme: $uri.path $uri.authority $uri.rawPath $uri.rawAuthority")
+    List<CidSerializable> search(String queryString) {
+
         def params = null
-        if (uri.query) {
-            params = uri.query.split('&').collectEntries {
-                it.split('=').collect { URLDecoder.decode(it, 'UTF-8') }
-            } as Map<String, String>
-        }
-        String key = uri.authority ? uri.authority + uri.path : uri.path
-        if (key == CidPath.SEPARATOR) {
-            searchAllFiles(params)
-        } else {
-            searchPath(key, params)
+        if (queryString) {
+            params = CidUtils.parseQuery(queryString)
         }
+        return searchAllFiles(params)
 
     }
 
-    private List<Object> searchAllFiles (Map<String,String> params) {
-        final results = new LinkedList<Object>()
-        final slurper = new JsonSlurper()
+    private List<CidSerializable> searchAllFiles (Map<String,String> params) {
+        final results = new LinkedList<CidSerializable>()
 
         Files.walkFileTree(metaLocation, new FileVisitor<Path>() {
 
@@ -137,10 +127,11 @@ class DefaultCidStore implements CidStore {
 
             @Override
             FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-
                 if (file.name.startsWith('.data.json') ) {
-                    final cidObject = slurper.parse( file.text.toCharArray() ) as Map
-                    DefaultCidStore.treatObject(cidObject, params, results)
+                    final cidObject = encoder.decode(file.text)
+                    if (CidUtils.checkParams(cidObject, params)){
+                        results.add(cidObject as CidSerializable)
+                    }
                 }
                 FileVisitResult.CONTINUE
             }
@@ -158,62 +149,4 @@ class DefaultCidStore implements CidStore {
 
         return results
     }
-
-    private List<Object> searchPath( String path, Map<String,String> params, String[] childs = []) {
-        final results = new LinkedList<Object>()
-        final slurper = new JsonSlurper()
-        final object = load(path)
-        if ( object ) {
-            final cidObject = slurper.parse(object.toString().toCharArray()) as Map
-            if (childs && childs.size() > 0) {
-                final output = cidObject.navigate(childs.join('.'))
-                if (output) {
-                    treatObject(output, params, results)
-                } else {
-                    throw new FileNotFoundException("Cid object $path/${childs ? childs.join('/') : ''} not found.")
-                }
-            } else {
-                treatObject(cidObject, params, results)
-            }
-        } else {
-            // If there isn't metadata check the parent to check if it is a subfolder of a task/workflow output
-            final currentPath = Path.of(path)
-            final parent = currentPath.getParent()
-            if( parent) {
-                ArrayList<String> newChilds = new ArrayList<String>()
-                newChilds.add(currentPath.getFileName().toString())
-                newChilds.addAll(childs)
-                return searchPath(parent.toString(), params, newChilds as String[])
-            } else {
-                throw new FileNotFoundException("Cid object $path/${childs ? childs.join('/') :''} not found.")
-            }
-        }
-        return results
-    }
-
-    protected static void treatObject(def object, Map<String,String> params, List<Object> results) {
-        if (params) {
-            if (object instanceof Collection) {
-                (object as Collection).forEach { treatObject(it, params, results) }
-            } else if (checkParams(object as Map, params)) {
-                results.add(object)
-            }
-        } else {
-            results.add(object)
-        }
-    }
-
-    private static boolean checkParams(Map object, Map<String,String> params) {
-        log.debug("Checking $object, $params")
-        for (final entry : params.entrySet()) {
-            final value = object.navigate(entry.key)
-            log.debug("comparing $value, $entry.value")
-            if (!value || value.toString() != entry.value.toString() ) {
-                return false
-            }
-        }
-        log.debug("Returning true")
-        return true
-    }
-
 }