enterprise blobber allocation tests.

.github/workflows/tokenomics_ci.yml

@@ -25,6 +25,25 @@ jobs:
         with:
           github-token: ${{ github.token }}
 
+      - name: 'Setup jq'
+        uses: dcarbone/[email protected]
+        with:
+          version: '1.7'
+          force: 'false'
+
+      - name: "Create Tenderly virtual testnet"
+        run: |
+          echo "TENDERLY_CREATION_INFO=$(curl -X POST \
+           -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \
+           -H "Content-Type: application/json" \
+           -d '{"slug":"mainnet-dev-${{ env.RUNNER_NUMBER }}-${{ github.run_id }}","displayName":"mainnet-dev-${{ env.RUNNER_NUMBER }}-${{ github.run_id }}","description":"","visibility":"TEAM","tags":{"purpose":"development"},"networkConfig":{"networkId":"1","blockNumber":"18512782","chainConfig":{"chainId":"1"},"baseFeePerGas":"1"},"explorerConfig":{"enabled":false,"verificationVisibility":"bytecode"},"syncState":false}' \
+           https://api.tenderly.co/api/v1/account/zus_network/project/project/testnet/container)" >> $GITHUB_ENV
+
+      - name: "Parse Tenderly virtual testnet creation transaction result"
+        run: |
+          echo "TENDERLY_VIRTUAL_TESTNET_ID=$(echo '${{ env.TENDERLY_CREATION_INFO }}' | jq -r '.container.id')" >> $GITHUB_ENV
+          echo "TENDERLY_VIRTUAL_TESTNET_RPC_ID=$(echo '${{ env.TENDERLY_CREATION_INFO }}' | jq -r '.container.connectivityConfig.endpoints[0].id')" >> $GITHUB_ENV
+
       - name: "Config: Run tests against existing 0Chain network"
         if: github.event_name == 'workflow_dispatch' && github.event.inputs.existing_network != ''
         run: |
@@ -44,9 +63,134 @@ jobs:
           echo "TOKENOMICS_SLASH_REWARD_TESTS=$(echo $(([ -z 'TestBlobberSlashPenalty' ] && echo '') || echo 'TestBlobberSlashPenalty'))" >> $GITHUB_ENV
           echo "TOKENOMICS_BLOCK_REWARD_TESTS=$(echo $(([ -z 'TestBlockRewardsForBlobbers' ] && echo '') || echo 'TestBlockRewardsForBlobbers'))" >> $GITHUB_ENV
           echo "TOKENOMICS_MIN_STAKE_TESTS=$(echo $(([ -z 'TestMinStakeForProviders' ] && echo '') || echo 'TestMinStakeForProviders'))" >> $GITHUB_ENV
+          echo "ENTERPRISE_BLOBBER_CREATE_ALLOCATION_TESTS=$(echo $(([ -z 'TestCreateEnterpriseAllocation' ] && echo '') || echo 'TestCreateEnterpriseAllocation'))" >> $GITHUB_ENV
+          echo "ENTERPRISE_BLOBBER_UPDATE_ALLOCATION_TESTS=$(echo $(([ -z 'TestUpdateEnterpriseAllocation' ] && echo '') || echo 'TestUpdateEnterpriseAllocation'))" >> $GITHUB_ENV
+          echo "ENTERPRISE_BLOBBER_REPLACE_ALLOCATION_TESTS=$(echo $(([ -z 'TestReplaceEnterpriseBlobberAllocation' ] && echo '') || echo 'TestReplaceEnterpriseBlobberAllocation'))" >> $GITHUB_ENV
+          echo "ENTERPRISE_BLOBBER_CANCEL_ALLOCATION_TESTS=$(echo $(([ -z 'TestCancelEnterpriseAllocation' ] && echo '') || echo 'TestCancelEnterpriseAllocation'))" >> $GITHUB_ENV
+          echo "ENTERPRISE_BLOBBER_FINALIZE_ALLOCATION_TESTS=$(echo $(([ -z 'TestFinalizeEnterpriseAllocation' ] && echo '') || echo 'TestFinalizeEnterpriseAllocation'))" >> $GITHUB_ENV
           echo "CLIENT_THROTTLING=$(echo $(([ -z 'TestClientThrottling' ] && echo '') || echo 'TestClientThrottling'))" >> $GITHUB_ENV
           echo "REPO_SNAPSHOTS_BRANCH=current-sprint" >> $GITHUB_ENV
 
+      - name: "Deploy 0Chain with enterprise blobbers"
+        if: github.event_name == 'push' || github.event.inputs.existing_network == ''
+        uses: 0chain/actions/deploy-0chain@feature/enterprise-blobber
+        with:
+          repo_snapshots_branch: "${{ env.REPO_SNAPSHOTS_BRANCH }}"
+          kube_config: ${{ secrets[format('DEV{0}KC', env.RUNNER_NUMBER)] }}
+          teardown_condition: "TESTS_PASSED"
+          SUBGRAPH_API_URL: ${{ secrets.SUBGRAPH_API_URL }}
+          TENDERLY_VIRTUAL_TESTNET_RPC_ID: ${{ env.TENDERLY_VIRTUAL_TESTNET_RPC_ID }}
+          graphnode_sc: ${{ secrets.GRAPHNODE_SC }}
+          graphnode_network: ${{ secrets.GRAPHNODE_NETWORK }}
+          graphnode_ethereum_node_url: https://virtual.mainnet.rpc.tenderly.co/${{ env.TENDERLY_VIRTUAL_TESTNET_RPC_ID }}
+          svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }}
+
+      - name: "Run Enterprise blobber create allocation tests"
+        uses: 0chain/actions/run-system-tests-tokenomics@master
+        with:
+          repo_snapshots_branch: "${{ env.REPO_SNAPSHOTS_BRANCH }}"
+          network: ${{ env.NETWORK_URL }}
+          svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }}
+          deploy_report_page: true
+          system_tests_branch: feat/enterprise-blobber-tests
+          archive_results: true
+          run_flaky_tests: false
+          run_api_system_tests: false
+          run_cli_system_tests: false
+          run_tokenomics_system_tests: true
+          tokenomics_test_filter: ${{ env.ENTERPRISE_BLOBBER_CREATE_ALLOCATION_TESTS }}
+          run_smoke_tests: ${{ inputs.run_smoke_tests }}
+          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
+          TENDERLY_VIRTUAL_TESTNET_RPC_ID: ${{ env.TENDERLY_VIRTUAL_TESTNET_RPC_ID }}
+
+      - name: "Run Enterprise blobber update allocation tests"
+        uses: 0chain/actions/run-system-tests-tokenomics@master
+        with:
+          repo_snapshots_branch: "${{ env.REPO_SNAPSHOTS_BRANCH }}"
+          network: ${{ env.NETWORK_URL }}
+          svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }}
+          deploy_report_page: true
+          archive_results: true
+          run_flaky_tests: false
+          system_tests_branch: feat/enterprise-blobber-tests
+          run_api_system_tests: false
+          run_cli_system_tests: false
+          run_tokenomics_system_tests: true
+          tokenomics_test_filter: ${{ env.ENTERPRISE_BLOBBER_UPDATE_ALLOCATION_TESTS }}
+          TENDERLY_VIRTUAL_TESTNET_ID: ""
+          run_smoke_tests: ${{ inputs.run_smoke_tests }}
+          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
+          TENDERLY_VIRTUAL_TESTNET_RPC_ID: ${{ env.TENDERLY_VIRTUAL_TESTNET_RPC_ID }}
+
+      - name: "Run Enterprise blobber replace blobber allocation tests"
+        uses: 0chain/actions/run-system-tests-tokenomics@master
+        with:
+          repo_snapshots_branch: "${{ env.REPO_SNAPSHOTS_BRANCH }}"
+          network: ${{ env.NETWORK_URL }}
+          svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }}
+          deploy_report_page: true
+          archive_results: true
+          system_tests_branch: feat/enterprise-blobber-tests
+          run_flaky_tests: false
+          run_api_system_tests: false
+          run_cli_system_tests: false
+          run_tokenomics_system_tests: true
+          tokenomics_test_filter: ${{ env.ENTERPRISE_BLOBBER_REPLACE_ALLOCATION_TESTS }}
+          TENDERLY_VIRTUAL_TESTNET_ID: ""
+          run_smoke_tests: ${{ inputs.run_smoke_tests }}
+          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
+          TENDERLY_VIRTUAL_TESTNET_RPC_ID: ${{ env.TENDERLY_VIRTUAL_TESTNET_RPC_ID }}
+
+      - name: "Run Enterprise blobber cancel allocation tests"
+        uses: 0chain/actions/run-system-tests-tokenomics@master
+        with:
+          repo_snapshots_branch: "${{ env.REPO_SNAPSHOTS_BRANCH }}"
+          network: ${{ env.NETWORK_URL }}
+          svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }}
+          deploy_report_page: true
+          archive_results: true
+          run_flaky_tests: false
+          system_tests_branch: feat/enterprise-blobber-tests
+          run_api_system_tests: false
+          run_cli_system_tests: false
+          run_tokenomics_system_tests: true
+          tokenomics_test_filter: ${{ env.ENTERPRISE_BLOBBER_CANCEL_ALLOCATION_TESTS }}
+          TENDERLY_VIRTUAL_TESTNET_ID: ""
+          run_smoke_tests: ${{ inputs.run_smoke_tests }}
+          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
+          TENDERLY_VIRTUAL_TESTNET_RPC_ID: ${{ env.TENDERLY_VIRTUAL_TESTNET_RPC_ID }}
+
+      - name: "Run Enterprise blobber finalize allocation tests"
+        uses: 0chain/actions/run-system-tests-tokenomics@master
+        with:
+          repo_snapshots_branch: "${{ env.REPO_SNAPSHOTS_BRANCH }}"
+          network: ${{ env.NETWORK_URL }}
+          svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }}
+          deploy_report_page: true
+          system_tests_branch: feat/enterprise-blobber-tests
+          archive_results: true
+          run_flaky_tests: false
+          run_api_system_tests: false
+          run_cli_system_tests: false
+          run_tokenomics_system_tests: true
+          tokenomics_test_filter: ${{ env.ENTERPRISE_BLOBBER_FINALIZE_ALLOCATION_TESTS }}
+          TENDERLY_VIRTUAL_TESTNET_ID: ""
+          run_smoke_tests: ${{ inputs.run_smoke_tests }}
+          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
+          TENDERLY_VIRTUAL_TESTNET_RPC_ID: ${{ env.TENDERLY_VIRTUAL_TESTNET_RPC_ID }}
+
+      - name: "Remove Tenderly virtual testnet"
+        if: always()
+        run: |
+          curl -X DELETE \
+            -H "x-access-key: ${{ secrets.TENDERLY_SECRET }}" \
+            -H "Content-Type: application/json" \
+            https://api.tenderly.co/api/v1/account/zus_network/project/project/testnet/container/${{ env.TENDERLY_VIRTUAL_TESTNET_ID }}
 
       - name: "Deploy 0Chain"
         if: github.event_name == 'push' || github.event.inputs.existing_network == ''
@@ -261,8 +405,8 @@ jobs:
 
 
   notify_slack_on_failure:
-    runs-on: [self-hosted, arc-runner]
-    needs: [system-tests]
+    runs-on: [ self-hosted, arc-runner ]
+    needs: [ system-tests ]
     if: always() && (needs.system-tests.result == 'failure')
     steps:
       - name: "Notify Slack"
@@ -279,8 +423,8 @@ jobs:
           curl -X POST -H 'Content-type: application/json' --data "${payload}" ${{ secrets.DEVOPS_CHANNEL_WEBHOOK_URL }}
 
   notify_slack_on_success:
-    runs-on: [self-hosted, arc-runner]
-    needs: [system-tests]
+    runs-on: [ self-hosted, arc-runner ]
+    needs: [ system-tests ]
     if: always() && (needs.system-tests.result == 'success')
     steps:
       - name: "Notify Slack"

tests/api_tests/repair_allocation_test.go

@@ -353,18 +353,18 @@ func TestRepairSize(testSetup *testing.T) {
 	wallet := createWallet(t)
 	sdkClient.SetWallet(t, wallet)
 	apiClient.CreateReadPool(t, wallet, 0.5, client.TxSuccessfulStatus)
-	
-	t.RunSequentiallyWithTimeout("repair size in case of no blobber failure should be zero", 5 * time.Minute, func(t *test.SystemTest) {
+
+	t.RunSequentiallyWithTimeout("repair size in case of no blobber failure should be zero", 5*time.Minute, func(t *test.SystemTest) {
 		// create allocation with default blobber requirements
 		blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey)
 		allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus)
 		allocationID := apiClient.CreateAllocation(t, wallet, allocationBlobbers, client.TxSuccessfulStatus)
 		t.Logf("allocationID: %v", allocationID)
 
 		// create and upload a file of 2KB to allocation.
-		op := sdkClient.AddUploadOperation(t, "", "", int64(1024 * 2))
+		op := sdkClient.AddUploadOperation(t, "", "", int64(1024*2))
 		sdkClient.MultiOperation(t, allocationID, []sdk.OperationRequest{op})
-		
+
 		// assert both upload and download size should be zero
 		alloc, err := sdk.GetAllocation(allocationID)
 		require.NoErrorf(t, err, "allocation ID %v is not found", allocationID)
@@ -374,8 +374,8 @@ func TestRepairSize(testSetup *testing.T) {
 		require.Equal(t, uint64(0), rs.UploadSize, "upload size doesn't match")
 		require.Equal(t, uint64(0), rs.DownloadSize, "download size doesn't match")
 	})
-	
-	t.RunSequentiallyWithTimeout("repair size on single blobber failure should match", 5 * time.Minute, func(t *test.SystemTest) {
+
+	t.RunSequentiallyWithTimeout("repair size on single blobber failure should match", 5*time.Minute, func(t *test.SystemTest) {
 		// create allocation with default blobber requirements
 		blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey)
 		blobberRequirements.DataShards = 2
@@ -390,18 +390,18 @@ func TestRepairSize(testSetup *testing.T) {
 		alloc, err := sdk.GetAllocation(allocationID)
 		require.NoErrorf(t, err, "allocation ID %v is not found", allocationID)
 		alloc.Blobbers[0].Baseurl = "http://0zus.com/"
-		op := sdkClient.AddUploadOperation(t, "", "", int64(1024 * 2))
+		op := sdkClient.AddUploadOperation(t, "", "", int64(1024*2))
 		sdkClient.MultiOperation(t, allocationID, []sdk.OperationRequest{op}, client.WithRepair(alloc.Blobbers))
-		
+
 		// assert upload and download size should be 1KB and 2KB respectively
 		rs, err := alloc.RepairSize("/")
 		require.Nil(t, err)
 		t.Logf("repair size: %v", rs)
 		require.Equal(t, uint64(1024), rs.UploadSize, "upload size doesn't match")
-		require.Equal(t, uint64(1024 * 2), rs.DownloadSize, "download size doesn't match")
+		require.Equal(t, uint64(1024*2), rs.DownloadSize, "download size doesn't match")
 	})
 
-	t.RunSequentiallyWithTimeout("repair size with nested directories and two blobber failure should match", 5 * time.Minute, func(t *test.SystemTest) {
+	t.RunSequentiallyWithTimeout("repair size with nested directories and two blobber failure should match", 5*time.Minute, func(t *test.SystemTest) {
 		// create allocation with default blobber requirements
 		blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey)
 		blobberRequirements.DataShards = 2
@@ -419,31 +419,31 @@ func TestRepairSize(testSetup *testing.T) {
 		ops := []sdk.OperationRequest{
 			sdkClient.AddUploadOperationWithPath(t, allocationID, "/dir1/"),
 			sdkClient.AddUploadOperationWithPath(t, allocationID, "/dir1/"),
-			sdkClient.AddUploadOperationWithPath(t, allocationID, "/"),  
+			sdkClient.AddUploadOperationWithPath(t, allocationID, "/"),
 			sdkClient.AddUploadOperationWithPath(t, allocationID, "/"),
 		}
 		sdkClient.MultiOperation(t, allocationID, ops, client.WithRepair(alloc.Blobbers))
-		
+
 		// assert both upload and download size should be 2KB in /dir1
 		rs, err := alloc.RepairSize("/dir1")
 		require.Nilf(t, err, "error getting repair size in /dir1: %v", err)
 		t.Logf("repair size: %v", rs)
-		require.Equal(t, uint64(1024 * 2), rs.UploadSize, "upload size in directory /dir1 doesn't match")
-		require.Equal(t, uint64(1024 * 2), rs.DownloadSize, "download size in directory dir1 doesn't match")
-		
+		require.Equal(t, uint64(1024*2), rs.UploadSize, "upload size in directory /dir1 doesn't match")
+		require.Equal(t, uint64(1024*2), rs.DownloadSize, "download size in directory dir1 doesn't match")
+
 		// with trailing slash
-		// assert both upload and download size should be 2KB in /dir1/ 
+		// assert both upload and download size should be 2KB in /dir1/
 		rs, err = alloc.RepairSize("/dir1/")
 		require.Nilf(t, err, "error getting repair size in /dir1/: %v", err)
 		t.Logf("repair size: %v", rs)
-		require.Equal(t, uint64(1024 * 2), rs.UploadSize, "upload size in directory /dir1/ doesn't match")
-		require.Equal(t, uint64(1024 * 2), rs.DownloadSize, "download size in directory /dir1/ doesn't match")
-		
+		require.Equal(t, uint64(1024*2), rs.UploadSize, "upload size in directory /dir1/ doesn't match")
+		require.Equal(t, uint64(1024*2), rs.DownloadSize, "download size in directory /dir1/ doesn't match")
+
 		// assert both upload and download size should be 4KB in root directory
 		rs, err = alloc.RepairSize("/")
 		require.Nilf(t, err, "error getting repair size in /: %v", err)
 		t.Logf("repair size: %v", rs)
-		require.Equal(t, uint64(1024 * 4), rs.UploadSize, "upload size in root directory doesn't match")
-		require.Equal(t, uint64(1024 * 4), rs.DownloadSize, "download size in root directory doesn't match")
+		require.Equal(t, uint64(1024*4), rs.UploadSize, "upload size in root directory doesn't match")
+		require.Equal(t, uint64(1024*4), rs.DownloadSize, "download size in root directory doesn't match")
 	})
-}
+}

tests/cli_tests/sdk.go

@@ -50,21 +50,30 @@ func InitSDK(wallet, configFile string) error {
 	return err
 }
 
-// GetBlobberNotPartOfAllocation returns a blobber not part of current allocation
-func GetBlobberNotPartOfAllocation(walletname, configFile, allocationID string) (string, error) {
-	err := InitSDK(walletname, configFile)
+// GetBlobberIDNotPartOfAllocation returns a blobber not part of current allocation
+func GetBlobberIDNotPartOfAllocation(walletname, configFile, allocationID string) (string, error) {
+	blobber, err := getBlobberNotPartOfAllocation(walletname, configFile, allocationID)
+
 	if err != nil {
 		return "", err
 	}
+	return string(blobber.ID), err
+}
+
+func getBlobberNotPartOfAllocation(walletname, configFile, allocationID string) (*sdk.Blobber, error) {
+	err := InitSDK(walletname, configFile)
+	if err != nil {
+		return nil, err
+	}
 
 	a, err := sdk.GetAllocation(allocationID)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	blobbers, err := sdk.GetBlobbers(true, false)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	allocationBlobsMap := map[string]bool{}
@@ -74,11 +83,20 @@ func GetBlobberNotPartOfAllocation(walletname, configFile, allocationID string)
 
 	for _, blobber := range blobbers {
 		if _, ok := allocationBlobsMap[string(blobber.ID)]; !ok {
-			return string(blobber.ID), nil
+			return blobber, nil
 		}
 	}
 
-	return "", fmt.Errorf("failed to get blobber not part of allocation")
+	return nil, fmt.Errorf("failed to get blobber not part of allocation")
+}
+
+// GetBlobberIdAndUrlNotPartOfAllocation returns a blobber not part of current allocation
+func GetBlobberIdAndUrlNotPartOfAllocation(walletName, configFile, allocationID string) (blobberId, blobberUrl string, err error) {
+	blobber, err := getBlobberNotPartOfAllocation(walletName, configFile, allocationID)
+	if err != nil || blobber == nil {
+		return "", "", err
+	}
+	return string(blobber.ID), blobber.BaseURL, err
 }
 
 func generateRandomIndex(sliceLen int64) (*big.Int, error) {

tests/cli_tests/zboxcli_repair_test.go

@@ -16,8 +16,8 @@ import (
 
 func TestRepairSize(testSetup *testing.T) {
 	t := test.NewSystemTest(testSetup)
-	
-	t.RunSequentiallyWithTimeout("repair size should work", 5 * time.Minute, func(t *test.SystemTest) {
+
+	t.RunSequentiallyWithTimeout("repair size should work", 5*time.Minute, func(t *test.SystemTest) {
 		allocSize := int64(1 * MB)
 		fileSize := int64(512 * KB)
 
@@ -55,7 +55,7 @@ func TestRepairSize(testSetup *testing.T) {
 		require.Nilf(t, err, "error unmarshal repair size: %v", err)
 		require.Equal(t, uint64(0), rs.UploadSize, "upload size should be zero")
 		require.Equal(t, uint64(0), rs.DownloadSize, "download size should be zero")
-		
+
 		// optional repairpath
 		output, err = getRepairSize(t, configPath, map[string]interface{}{
 			"allocation": allocationID,
@@ -67,7 +67,6 @@ func TestRepairSize(testSetup *testing.T) {
 		require.Equal(t, uint64(0), rs2.UploadSize, "upload size should be zero")
 		require.Equal(t, uint64(0), rs2.DownloadSize, "download size should be zero")
 	})
-
 }
 
 func getRepairSize(t *test.SystemTest, cliConfigFilename string, param map[string]interface{}, retry bool) ([]string, error) {
@@ -90,4 +89,4 @@ func getRepairSizeForWallet(t *test.SystemTest, wallet, cliConfigFilename string
 	} else {
 		return cliutils.RunCommandWithoutRetry(cmd)
 	}
-}
+}