merged 1.1.0_mac

Author: Martin Metsoja

build/entitlements.mac.plist

@@ -6,6 +6,10 @@
     <true/>
     <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
     <true/>
+    <key>com.apple.security.cs.allow-dyld-environment-variables</key>
+    <true/>
+    <key>com.apple.security.cs.disable-library-validation</key>
+    <true/>
     <key>com.apple.security.cs.debugger</key>
     <true/>
     <key>com.apple.security.network.client</key>

gitlog.txt

@@ -1,15 +1,16 @@
-FROM ubuntu:24.04
+FROM --platform=linux/amd64 ubuntu:24.04
 ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8
 ENV CONDA_DIR=/opt/conda
 ENV PATH=$CONDA_DIR/bin:$PATH
-
-COPY image_development/protaxFungi.zip image_development/protaxAnimal.zip image_development/OptimOTU_v5.yaml /
-
 RUN apt-get update && \
     apt-get install -y wget git zip build-essential zlib1g-dev && \
-    git clone --branch main https://github.com/brendanf/optimotu_targets.git /optimotu_targets && \
-    sed -i 's|BASEDIR=$(pwd)|BASEDIR="/optimotu_targets"|g' /optimotu_targets/scripts/runprotax && \
+    git clone --branch main https://github.com/brendanf/optimotu_targets.git /optimotu_targets
+COPY image_development/protaxFungi/ /optimotu_targets/protaxFungi/
+COPY image_development/protaxAnimal/ /optimotu_targets/protaxAnimal/
+COPY image_development/OptimOTU_v5.yaml /
+
+RUN sed -i 's|BASEDIR=$(pwd)|BASEDIR="/optimotu_targets"|g' /optimotu_targets/scripts/runprotax && \
     sed -i '1i#!/bin/bash' /optimotu_targets/scripts/runprotax && \
     sed -i 's|$PROTAX/usearch10.0.240_i86linux32|usearch|g' /optimotu_targets/scripts/runprotax && \
     cd /optimotu_targets && \
@@ -18,7 +19,6 @@ RUN apt-get update && \
     chmod +x bin/fastqindex_0.9.0b && \
     wget https://s3.hpc.ut.ee/plutof-public/original/5dcbe93a-b50c-4ff5-8898-528160c4e593.zip && \
     unzip -j 5dcbe93a-b50c-4ff5-8898-528160c4e593.zip data/shs_out.txt data/sanger_refs_sh.fasta -d data/sh_matching_data && \
-    rm -rf protaxFungi && \
     wget -O FinPROTAX.zip https://github.com/psomervuo/FinPROTAX/archive/refs/heads/main.zip && \
     unzip FinPROTAX.zip -d . && \
     mv FinPROTAX-main FinPROTAX && \
@@ -28,13 +28,10 @@ RUN apt-get update && \
     cd protaxAnimal_bin/protaxA_c2 && make && \
     cp classify_v2 classify_info dist_best dist_matrix dist_bipart /optimotu_targets/bin && \
     cd /optimotu_targets && \
-    unzip /protaxFungi.zip -d . && \
-    unzip /protaxAnimal.zip -d . && \
-    gzip protaxAnimal/refs.aln && \
-    mv protaxAnimal/refs.aln.gz protaxAnimal/refs.aln && \
     wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
     /bin/bash ~/miniconda.sh -b -p /opt/conda && \
     conda env create -f /OptimOTU_v5.yaml && \
+    chmod -R 777 /opt/conda/envs/OptimOTU_v5 && \
     eval "$(conda shell.bash hook)" && \
     conda activate OptimOTU_v5 && \
     Rscript -e "install.packages('renv', repos='https://cloud.r-project.org')" && \
@@ -47,8 +44,6 @@ RUN apt-get update && \
     usearch -makeudb_usearch protaxFungi/addedmodel/amptk_synmock.fasta -output protaxFungi/addedmodel/amptk_synmock.udb && \
     usearch -makeudb_usearch protaxFungi/addedmodel/its2.fa -output protaxFungi/addedmodel/its2.udb && \
     rm -f /optimotu_targets/5dcbe93a-b50c-4ff5-8898-528160c4e593.zip && \
-    rm -f /protaxFungi.zip && \
-    rm -f /protaxAnimal.zip && \
     rm -f /optimotu_targets/FinPROTAX.zip && \
     rm -f /optimotu_targets/protaxAnimal_bin.zip && \
     rm -f ~/miniconda.sh && \
@@ -57,4 +52,6 @@ RUN apt-get update && \
     apt-get remove -y wget git build-essential && \
     apt-get autoremove -y && \
     apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/* && \
+    gzip protaxAnimal/refs.aln && \
+    mv protaxAnimal/refs.aln.gz protaxAnimal/refs.aln

image_development/Dockerfile_optimOTU2

@@ -25,7 +25,8 @@ RUN apt-get update -y \
     && apt-get install -y libdw1 \
     && apt-get install -y libuv1 \
     && apt-get install libnghttp2-14 \
-    && apt-get install locales -y
+    && apt-get install locales -y \
+    && apt-get install libgit2-dev -y
 RUN wget https://github.com/shenwei356/seqkit/releases/download/v2.9.0/seqkit_linux_amd64.tar.gz -O /tmp/seqkit.tar.gz && \
     tar zxvf /tmp/seqkit.tar.gz -C /usr/bin/ && rm /tmp/seqkit.tar.gz
 RUN git clone --branch v2.29.4 https://github.com/torognes/vsearch.git \
@@ -41,7 +42,7 @@ RUN Rscript -e 'install.packages("BiocManager")'
 RUN Rscript -e 'BiocManager::install(version = "3.20", ask=FALSE)'
 RUN Rscript -e 'BiocManager::install("dada2")'
 RUN Rscript -e 'remotes::install_github("tobiasgf/lulu")'
-RUN wget ftp://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/ORFfinder/linux-i64/ORFfinder.gz
+RUN wget https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/ORFfinder/linux-i64/ORFfinder.gz
 RUN gunzip ORFfinder.gz
 RUN chmod 755 ORFfinder
 RUN mv ORFfinder /bin/.

image_development/Dockerfile_vsearch_dada2_v3

@@ -244,7 +244,7 @@ export default {
               let result = await dockerode
                 .run(
                   step.imageName,
-                  ["sh", "-c", `chmod +x /scripts/${scriptName} && /scripts/${scriptName}`],
+                  ["bash", "-c", `bash /scripts/${scriptName}`],
                   [stdout, stderr],
                   dockerProps
                 )
@@ -382,7 +382,7 @@ export default {
             let result = await dockerode
               .run(
                 selectedStep.imageName,
-                ["sh", "-c", `chmod +x /scripts/${selectedStep.scriptName} && /scripts/${selectedStep.scriptName}`],
+                ["bash", "-c", `bash /scripts/${selectedStep.scriptName}`],
                 [stdout, stderr],
                 dockerProps
               )
@@ -762,27 +762,24 @@ export default {
           await this.$store.dispatch('generateOptimOTUYamlConfig');
           await this.imageCheck('pipecraft/optimotu:5');
           await this.clearContainerConflicts('optimotu');
-          let logStream;
           try {            
-            const logDir = this.$store.state.inputDir;
-            const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-            const logFile = path.join(logDir, `optimotu-${timestamp}.log`);
-            logStream = fs.createWriteStream(logFile, { 
-              flags: 'a',
-              mode: 0o666  // Read/write permissions for all users
-            });
             const container = await dockerode.createContainer({
               Image: 'pipecraft/optimotu:5',
               name: 'optimotu',
               Cmd: ['/scripts/run_optimotu.sh'],
               Tty: false,
               AttachStdout: true,
               AttachStderr: true,
+              Platform: "linux/amd64",
               Env: [
                 `HOST_UID=${this.userId}`,
                 `HOST_GID=${this.groupId}`,
                 `fileFormat=${this.$store.state.data.fileFormat}`,
-                `readType=${this.$store.state.data.readType}`
+                `readType=${this.$store.state.data.readType}`,
+                'R_ENABLE_JIT=0',                    // Disable JIT compilation
+                'R_COMPILE_PKGS=0',                  // Disable package compilation
+                'R_DISABLE_BYTECODE=1',              // Disable bytecode compilation
+                'R_KEEP_PKG_SOURCE=yes'              // Keep package sources
               ],
               HostConfig: {
                 Binds: this.getOptimOTUBinds(),
@@ -798,19 +795,15 @@ export default {
             });
     
             stream.on('data', (data) => {
-              const output = data.toString();
-              console.log(output);
-              logStream.write(output);
+              console.log(data.toString());
             });
             
             await container.start();
     
             const data = await container.wait();
             console.log('Container exited with status code:', data.StatusCode);
             this.$store.commit("resetRunInfo");
-            // Close the log stream
-            logStream.end();
-            console.log(`Container log written to ${logFile}`);
+
             if (data.StatusCode == 0) {
               Swal.fire("Workflow finished");
             } else {
@@ -823,10 +816,6 @@ export default {
             await container.remove({ v: true, force: true });
           } catch (err) {
             console.error('Error running container:', err);
-            if (logStream) {
-              logStream.write(`\nERROR: ${err.message}\n${err.stack}\n`);
-              logStream.end();
-            }
             this.$store.commit("resetRunInfo");
             
             // Check if the error is due to the user stopping the container
@@ -883,7 +872,7 @@ export default {
             dockerode
               .run(
                 "vmikk/nextits:0.5.0",
-                ["sh", "-c", `/scripts/NextITS_Pipeline.sh`],
+                ["bash", "-c", `bash /scripts/NextITS_Pipeline.sh`],
                 false,
                 props,
                 (err, data, container) => {

src/components/Run.vue

@@ -21,22 +21,22 @@ trimming:
   cut_R2: 0
   action: "trim"
 protax:
-  aligned: yes
-  location: "/optimotu_targets/protaxAnimal"
+  aligned: no
+  location: "/optimotu_targets/protaxFungi"
   ranks:
-    - kingdom: Animalia
-    - phylum: Arthropoda
+    - kingdom: Fungi
+    - phylum
     - class
     - order
     - family
     - genus
     - species
-orient: fwd
+orient: mixed
 control:
   spike:
   positive:
-forward_primer: "CCHGAYATRGCHTTYCCHCG"
-reverse_primer: "TCDGGRTGNCCRAARAAYCA"
+forward_primer: "GCATCGATGAAGAACGCAGC"
+reverse_primer: "TCCTCCGCTTATTGATATGC"
 custom_sample_table: FALSE
 filtering:
   maxEE_R1: 1
@@ -45,14 +45,15 @@ tag_jump:
   f: 0.03
   p: 1
 amplicon_model:
-  model_type: HMM
-  model_file: /optimotu_targets/data/COI.hmm
-  model_align: yes
-  numt_filter: yes
+  model_type: CM
+  model_file: /optimotu_targets/data/ITS3_ITS4.cm
+  model_align: no
+  numt_filter: no
   model_filter:
-    max_model_start: 245
-    min_model_end: 652
-    min_model_score: 200
+    max_model_start: 5
+    min_model_end: 140
+    min_model_score: 50
 outgroup_reference:
-  sequences: /optimotu_targets/data/outgroup/BOLD_Public.27-Sep-2024.part_001.m300.fasta
-cluster_thresholds: /optimotu_targets/metadata/MBRAVE_thresholds.tsv
+  sequences: /optimotu_targets/data/sh_matching_data/sanger_refs_sh.fasta
+  taxonomy: /optimotu_targets/data/sh_matching_data/shs_out.txt
+cluster_thresholds: /optimotu_targets/metadata/GSSP_thresholds.tsv

src/pipecraft-core/service_scripts/pipeline_options.yaml

@@ -10,6 +10,8 @@ export fileFormat
 
 echo "specified fileFormat: $fileFormat"
 
+
+
 # # check files in 01_raw and subdirectories
 # # Find all subdirectories in 01_raw
  while IFS= read -r subdir; do
@@ -42,6 +44,20 @@ cp -f /scripts/pipeline_options.yaml /optimotu_targets/sequences/pipeline_option
 # Activate the conda environment
 source /opt/conda/etc/profile.d/conda.sh
 conda activate OptimOTU_v5
+
+# Try to install a compatible version of qs2
+echo "Installing compatible version of qs2..."
+R --vanilla -e '
+  # Remove existing qs2 if present
+  if("qs2" %in% installed.packages()[,"Package"]) remove.packages("qs2")
+  
+  # Install from source with minimal optimizations
+  Sys.setenv(PKG_CFLAGS="-O0 -march=x86-64")
+  Sys.setenv(PKG_CXXFLAGS="-O0 -march=x86-64")
+  install.packages("qs2", type="source", repos="https://cloud.r-project.org")
+'
+
+
 cd /optimotu_targets
 
 # Print current environment for debugging
@@ -94,12 +110,7 @@ Start time: $start_time
 End time: $(date)
 Runtime: $runtime seconds
 
-echo "All operations completed."
 
-if [ ! -z "$HOST_UID" ] && [ ! -z "$HOST_GID" ]; then
-  echo "Setting ownership of /optimotu_targets to $HOST_UID:$HOST_GID"
-  chown -R $HOST_UID:$HOST_GID /optimotu_targets/sequences
-fi
 
 The outputs of the pipeline are a set of tables in TSV format (tab-delimited files) and 
 RDS format (for easy loading in R), as well as sequences in gzipped FASTA format.
@@ -127,4 +138,11 @@ EOF
 
 #Done
 printf "\nDONE "
-printf "Total time: $runtime sec.\n "
+printf "Total time: $runtime sec.\n "
+
+echo "All operations completed."
+
+if [ ! -z "$HOST_UID" ] && [ ! -z "$HOST_GID" ]; then
+  echo "Setting ownership of /optimotu_targets to $HOST_UID:$HOST_GID"
+  chown -R $HOST_UID:$HOST_GID /optimotu_targets/sequences
+fi