Merge pull request #43 from x1wins/feature/docker-machine

Feature/docker machine

DOCKER-SWARM.md

@@ -42,6 +42,16 @@ docker-machine create -d virtualbox worker2
 docker-machine ls
 ```
 
+aws
+- https://blog.scottlowe.org/2016/03/22/using-docker-machine-with-aws/
+```
+docker-machine create -d amazonec2 \
+--amazonec2-region us-west-1 \
+--amazonec2-instance-type "t2.micro" \
+--amazonec2-ssh-keypath ~/.ssh/ssh_key \
+aws-test
+```
+
 > if you don't have virtualbox. you got blow of message.
 ```
 VBoxManage not found. Make sure VirtualBox is installed and VBoxManage is in the path.
@@ -63,6 +73,12 @@ To add a worker to this swarm, run the following command:
 To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
 ```
 
+if you want get token for worker again
+- https://docs.docker.com/engine/reference/commandline/swarm_join-token/
+```
+docker swarm join-token worker
+```
+
 ```
 docker-machine ssh worker1
    ( '>')
@@ -89,8 +105,14 @@ This node joined a swarm as a worker.
 ```
 docker-machine ssh master
 
-docker service create --name registry --publish published=5000,target=5000 registry:2
+docker service create --name registry --insecure-registry=[myip]:5000 --constraint 'node.role == manager' --publish published=5000,target=5000 registry:2
 curl http://localhost:5000/v2/
+
+sudo vim /etc/docker/daemon.json
+{"insecure-registries":["your ip:5000"] }
+sudo service docker restart
+
+https://github.com/docker/distribution/issues/1874
 ```
 
 ## Initial source
@@ -107,7 +129,7 @@ git checkout master # or feature/docker-stack
 cd CW-OVP/
 git pull
 git checkout master
-git reset --hard origin/master # or origin/feature/docker-stack
+git reset --hard origin/master # or origin/feature/docker-machine
 ```
 
 ## Update s3
@@ -178,6 +200,7 @@ docker service update --image 127.0.0.1:5000/cw-ovp CW-OVP_web
 docker service ps --no-trunc {serviceName}
 docker service ps --no-trunc t9zabkgynr8c
 docker service ps --no-trunc kged4le7e3jn
+
 mkdir /home/docker/CW-OVP/tmp/redis                                                                                                                                               
 mkdir /home/docker/CW-OVP/tmp/db
 ```

app/controllers/encodes_controller.rb

@@ -30,8 +30,6 @@ def create
     respond_to do |format|
       if @encode.save
         Message::Send.call(Message::Event::CREATED, Message::Body.new(@encode, nil, nil, @encode.file.filename, nil))
-        ThumbnailWorker.perform_async(@encode.id)
-        EncodeWorker.perform_async(@encode.id)
         format.html { redirect_to @encode, notice: 'Encode was successfully created.' }
         format.json { render :show, status: :created, location: @encode }
       else

app/encoding/cp.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+#aws s3 cp /storage/encode/2020/10/09/44/thumbnail s3://vod-origin/encode/2020/10/09/44/thumbnail --recursive
+aws s3 cp $3 s3://$1/$2

app/encoding/rm.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+rm -rf $3

app/models/encode.rb

@@ -8,9 +8,15 @@ class Encode < ApplicationRecord
   scope :published, -> { where(published: true) }
   scope :by_date, -> { order('id DESC') }
   before_create :default_values
+  after_create_commit :run_worker
   VALIDATED_FILE_EXTENSIONS = %w( .ts .mp4 .mov .avi .mkv )
   THUMBNAIL_COUNT = 10
 
+  def run_worker
+    ThumbnailWorker.perform_async(self.id)
+    EncodeWorker.perform_async(self.id)
+  end
+
   def default_values
     self.started_at ||= Time.now
     self.published ||= true

app/workers/encode_worker.rb

@@ -1,6 +1,6 @@
 class EncodeWorker
   include Sidekiq::Worker
-  sidekiq_options retry: 3
+  sidekiq_options retry: false, backtrace: true
 
   def perform(encode_id)
     encode = Encode.find(encode_id)

app/workers/thumbnail_worker.rb

@@ -1,6 +1,6 @@
 class ThumbnailWorker
   include Sidekiq::Worker
-  sidekiq_options retry: false # job will be discarded if it fails
+  sidekiq_options retry: false, backtrace: true
 
   def perform(encode_id)
     encode = Encode.find(encode_id)
@@ -20,22 +20,28 @@ def perform(encode_id)
           thumbnail_url = Storage::Url::Full::Thumbnail.call(encode, base_url, thumbnail_filename)
           encode.assets.create(format: 'image', url: thumbnail_url)
           encode.thumbnails.attach(io: File.open(thumbnail_file_full_path), filename: thumbnail_filename, content_type: "image/png")
-          thumbnail_rails_url = Rails.application.routes.url_helpers.rails_blob_path(encode.thumbnails.last, disposition: "attachment", only_path: true)
           message = "Extracted #{i}th Thumbnail"
-          Message::Send.call(Message::Event::THUNBNAIL_PROCESSING, Message::Body.new(encode, nil, message, nil, thumbnail_rails_url))
+
+          cdn_bucket = ENV['CDN_BUCKET']
+          thumbnail_relative_path = Storage::Url::Relative::Thumbnail.call(encode)
+          thumbnail_local_full_path = Storage::Path::Local::Thumbnail.call(encode)
+          cdn_file_path = "#{thumbnail_relative_path}/#{thumbnail_filename}"
+          cp_thumbnail_to_cdn_cmd = `sh app/encoding/cp.sh #{cdn_bucket} "#{cdn_file_path}" #{thumbnail_file_full_path}`
+          Sidekiq.logger.info " cp_thumbnail_to_cdn_cmd #{cdn_bucket} #{cdn_file_path} #{thumbnail_file_full_path}"
+
+          Message::Send.call(Message::Event::THUNBNAIL_PROCESSING, Message::Body.new(encode, nil, message, nil, thumbnail_url))
           if i == Encode::THUMBNAIL_COUNT
             message = "Completed Extracting Thumbnail"
-            first_thumbnail_rails_url = Rails.application.routes.url_helpers.rails_blob_path(encode.thumbnails.first, disposition: "attachment", only_path: true)
-            Message::Send.call(Message::Event::THUMBNAIL_RAILS_URL, Message::Body.new(encode, nil, message, nil, first_thumbnail_rails_url))
+            Message::Send.call(Message::Event::THUMBNAIL_RAILS_URL, Message::Body.new(encode, nil, message, nil, thumbnail_url))
           end
         end
       end
 
       cdn_bucket = ENV['CDN_BUCKET']
       thumbnail_relative_path = Storage::Url::Relative::Thumbnail.call(encode)
       thumbnail_local_full_path = Storage::Path::Local::Thumbnail.call(encode)
-      move_thumbnail_to_cdn_cmd = `sh app/encoding/mv.sh #{cdn_bucket} #{thumbnail_relative_path} #{thumbnail_local_full_path}`
-      Sidekiq.logger.debug "move_thumbnail_to_cdn_cmd : #{move_thumbnail_to_cdn_cmd}"
+      rm_thumbnail_to_cdn_cmd = `sh app/encoding/rm.sh #{cdn_bucket} #{thumbnail_relative_path} #{thumbnail_local_full_path}`
+
       thumbnail_cdn_url = ""
       for asset in encode.assets
         thumbnail_cdn_url += (asset.url+"<br/>") if asset.format == 'image'

docker-stack.yml

@@ -65,18 +65,11 @@ services:
         condition: on-failure
       placement:
         constraints: [node.role == worker]
-  visualizer:
-    image: dockersamples/visualizer:stable
-    ports:
-      - "8080:8080"
-    stop_grace_period: 1m30s
     volumes:
-      - "/var/run/docker.sock:/var/run/docker.sock"
-    deploy:
-      placement:
-        constraints: [node.role == manager]
+      - local-storage:/storage
 networks:
   backend:
 volumes:
   db-data:
-  redis-data:
+  redis-data:
+  local-storage: