Add support for for dynamic data updates

- use the new LeanerCloud instance-type-info API for instance type data updates.
- fallback to static data and previous version of the updated data in case of corrupt new data or data that fails to unmarshal with the deployed data structure.

README.md

@@ -29,6 +29,8 @@ go get -u github.com/LeanerCloud/ec2-instances-info/...
 
 ## Usage
 
+### One-off usage, with static data
+
 ```golang
 import "github.com/LeanerCloud/ec2-instances-info"
 
@@ -42,6 +44,42 @@ for _, i := range *data {
 
 See the examples directory for a working code example.
 
+### One-off usage, with updated instance type data
+
+```golang
+import "github.com/LeanerCloud/ec2-instances-info"
+
+key := "API_KEY" // API keys are available upon demand from [email protected], free of charge for personal use
+
+err:= ec2instancesinfo.UpdateData(nil, &key);
+if err!= nil{
+   fmt.Println("Couldn't update instance type data, reverting to static compile-time data", err.Error())
+}
+
+data, err := ec2instancesinfo.Data() // needs to be called once after data updates
+
+// This would print all the available instance type names:
+for _, i := range *data {
+  fmt.Println("Instance type", i.InstanceType)
+}
+```
+
+### Continuous usage, with instance type data updated every 2 days
+
+```golang
+import "github.com/LeanerCloud/ec2-instances-info"
+
+key := "API_KEY"
+go ec2instancesinfo.Updater(2, nil, &key); // use 0 or negative values for weekly updates
+
+data, err := ec2instancesinfo.Data() // only needed once
+
+// This would print all the available instance type names:
+for _, i := range *data {
+  fmt.Println("Instance type", i.InstanceType)
+}
+```
+
 ## Contributing
 
 Pull requests and feedback are welcome.

ec2_instance_data.go

@@ -7,6 +7,7 @@ package ec2instancesinfo
 import (
 	_ "embed"
 	"encoding/json"
+	"log"
 	"sort"
 	"strconv"
 	"strings"
@@ -45,13 +46,9 @@ type jsonInstance struct {
 	MaxBandwidth float32 `json:"max_bandwidth"`
 	InstanceType string  `json:"instance_type"`
 
-	// ECU is ignored because it's useless and also unreliable when parsing the
-	// data structure: usually it's a number, but it can also be the string
-	// "variable"
-	// ECU float32 `json:"ECU"`
-
-	Memory          float32 `json:"memory"`
-	EBSMaxBandwidth float32 `json:"ebs_max_bandwidth"`
+	Memory          float32
+	MemoryRaw       json.RawMessage `json:"memory"`
+	EBSMaxBandwidth float32         `json:"ebs_max_bandwidth"`
 }
 
 type StorageConfiguration struct {
@@ -98,7 +95,9 @@ type Reserved struct {
 }
 
 //go:embed data/instances.json
-var dataFile []byte
+var staticDataBody []byte
+
+var dataBody, backupDataBody []byte
 
 //------------------------------------------------------------------------------
 
@@ -114,20 +113,42 @@ func Data() (*InstanceData, error) {
 
 	var d InstanceData
 
-	err := json.Unmarshal(dataFile, &d)
-	if err != nil {
-		return nil, errors.Errorf("couldn't read the data asset: %s", err.Error())
+	if len(dataBody) > 0 {
+		log.Println("We have updated data, trying to unmarshal it")
+		err := json.Unmarshal(dataBody, &d)
+		if err != nil {
+			log.Printf("couldn't unmarshal the updated data, reverting to the backup data : %s", err.Error())
+			err := json.Unmarshal(backupDataBody, &d)
+			if err != nil {
+				return nil, errors.Errorf("couldn't unmarshal backup data: %s", err.Error())
+			}
+			backupDataBody = []byte{}
+		}
+	} else {
+		log.Println("Using the static instance type data")
+		err := json.Unmarshal(staticDataBody, &d)
+		if err != nil {
+			return nil, errors.Errorf("couldn't unmarshal data: %s", err.Error())
+		}
+
 	}
 
 	// Handle "N/A" values in the VCPU field for i3.metal instance type and
 	// string ("variable") and integer values in the ECU field
 	for i := range d {
 		var vcpu, intECU int
 		var stringECU string
-		if err = json.Unmarshal(d[i].VCPURaw, &vcpu); err == nil {
+		var memory float32
+
+		if err := json.Unmarshal(d[i].VCPURaw, &vcpu); err == nil {
 			d[i].VCPU = vcpu
 		}
-		if err = json.Unmarshal(d[i].ECURaw, &intECU); err == nil {
+
+		if err := json.Unmarshal(d[i].MemoryRaw, &memory); err == nil {
+			d[i].Memory = memory
+		}
+
+		if err := json.Unmarshal(d[i].ECURaw, &intECU); err == nil {
 			d[i].ECU = strconv.Itoa(intECU)
 		} else if err = json.Unmarshal(d[i].ECURaw, &stringECU); err == nil {
 			d[i].ECU = stringECU

examples/update-daily/update-daily.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	ec2instancesinfo "github.com/LeanerCloud/ec2-instances-info"
+)
+
+func main() {
+
+	key := "API_KEY" // keys available from [email protected]
+
+	go ec2instancesinfo.Updater(2, nil, &key) // use 0 or negative values for weekly updates
+
+	time.Sleep(30 * time.Second)
+
+	data, err := ec2instancesinfo.Data()
+
+	if err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+
+	for _, i := range *data {
+		fmt.Print(
+			"Instance type: ", i.InstanceType,
+			",\tCPU: ", i.PhysicalProcessor,
+			",\t Arch: ", i.Arch[0],
+			",\tCPU cores: ", i.VCPU,
+			",\tMemory(GB): ", i.Memory,
+			",\tEBS Throughput(MB/s): ", i.EBSThroughput,
+			",\tLinux OD cost in us-east-1: ", i.Pricing["us-east-1"].Linux.OnDemand,
+			",\tWindows OD cost in us-east-1: ", i.Pricing["us-east-1"].MSWin.OnDemand,
+			",\tRHEL OD cost in us-east-1: ", i.Pricing["us-east-1"].RHEL.OnDemand,
+			",\tSLES OD cost in us-east-1: ", i.Pricing["us-east-1"].SLES.OnDemand,
+			",\tLinux Spot cost in us-east-1: ", i.Pricing["us-east-1"].Linux.SpotMin,
+			",\tLinux Standard RI 1y AllUpfront cost in us-east-1: ", i.Pricing["us-east-1"].Linux.Reserved.StandardAllUpfront1Year)
+		if i.Storage != nil {
+			fmt.Print(",\tLocal storage volume size(GB): ", i.Storage.Size,
+				",\tLocal storage volumes: ", i.Storage.Devices,
+				",\tLocal storage SSD: ", i.Storage.SSD)
+		}
+
+		fmt.Println(",\tEBS surcharge: ", i.Pricing["us-east-1"].EBSSurcharge)
+	}
+
+}

examples/update-one-off/one-off.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	ec2instancesinfo "github.com/LeanerCloud/ec2-instances-info"
+)
+
+func main() {
+
+	key := "API_KEY" // keys available from [email protected]
+	err := ec2instancesinfo.UpdateData(nil, &key)
+	if err != nil {
+		fmt.Println("Couldn't update instance type data, reverting to static compile-time data", err.Error())
+	}
+
+	data, err := ec2instancesinfo.Data()
+
+	if err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+
+	for _, i := range *data {
+		fmt.Print(
+			"Instance type: ", i.InstanceType,
+			",\tCPU: ", i.PhysicalProcessor,
+			",\t Arch: ", i.Arch[0],
+			",\tCPU cores: ", i.VCPU,
+			",\tMemory(GB): ", i.Memory,
+			",\tEBS Throughput(MB/s): ", i.EBSThroughput,
+			// ",\tLinux OD cost in us-east-1: ", i.Pricing["us-east-1"].Linux.OnDemand,
+			// ",\tWindows OD cost in us-east-1: ", i.Pricing["us-east-1"].MSWin.OnDemand,
+			// ",\tRHEL OD cost in us-east-1: ", i.Pricing["us-east-1"].RHEL.OnDemand,
+			// ",\tSLES OD cost in us-east-1: ", i.Pricing["us-east-1"].SLES.OnDemand,
+			// ",\tLinux Spot cost in us-east-1: ", i.Pricing["us-east-1"].Linux.SpotMin,
+			// ",\tLinux Standard RI 1y AllUpfront cost in us-east-1: ", i.Pricing["us-east-1"].Linux.Reserved.StandardAllUpfront1Year
+		)
+		if i.Storage != nil {
+			fmt.Print(",\tLocal storage volume size(GB): ", i.Storage.Size,
+				",\tLocal storage volumes: ", i.Storage.Devices,
+				",\tLocal storage SSD: ", i.Storage.SSD)
+		}
+
+		fmt.Println(",\tEBS surcharge: ", i.Pricing["us-east-1"].EBSSurcharge)
+	}
+
+}

updater.go

@@ -0,0 +1,130 @@
+package ec2instancesinfo
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"time"
+)
+
+const (
+	defaultAPIHost         = "api.leanercloud.com"
+	defaultAPIPath         = "/instance-type-info/"
+	defaultRefreshInterval = 7
+)
+
+func getDataURL(apiHost, apiKey string) (*string, error) {
+	url := fmt.Sprintf("https://%s%s", apiHost, defaultAPIPath)
+
+	client := &http.Client{}
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error creating request: %s", err)
+	}
+
+	req.Header.Set("Authorization", "Bearer "+apiKey)
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error sending request: %s", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("received unexpected HTTP status: %s", resp.Status)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error reading response body: %s", err)
+	}
+	ret := string(body)
+
+	return &ret, nil
+}
+
+func UpdateData(apiHost, apiKey *string) error {
+	if apiHost == nil {
+		s := defaultAPIHost
+		apiHost = &s
+	}
+
+	log.Printf("Dynamic data size before: %d, downloading new instance type data.", len(dataBody))
+
+	url, err := getDataURL(*apiHost, *apiKey)
+
+	if err != nil {
+		return fmt.Errorf("error getting download URL file: %s", err)
+	}
+
+	client := &http.Client{}
+	req, err := http.NewRequest("GET", *url, nil)
+	if err != nil {
+		return fmt.Errorf("error creating request: %s", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error sending request: %s", err)
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("error reading response body: %s", err)
+	}
+	if len(dataBody) > 0 {
+		backupDataBody = dataBody
+	} else {
+		backupDataBody = staticDataBody
+	}
+
+	dataBody = body
+	log.Println("Data size after:", len(dataBody))
+
+	out, err := os.Create("instances_dump.json")
+	if err != nil {
+		panic("Couldn't create dump file")
+	}
+
+	defer out.Close()
+	out.Write(body)
+
+	return nil
+}
+
+func Updater(refreshDays int, apiHost, apiKey *string) error {
+	if apiKey == nil {
+		log.Println("API key is missing")
+		return fmt.Errorf("API key is missing")
+	}
+
+	if apiHost == nil {
+		host := defaultAPIHost
+		apiHost = &host
+	}
+
+	if refreshDays <= 0 {
+		refreshDays = defaultRefreshInterval
+	}
+	refreshInterval := time.Duration(refreshDays) * 24 * time.Hour
+
+	if err := UpdateData(apiHost, apiKey); err != nil {
+		log.Printf("Failed to download updated data: %s", err.Error())
+		return fmt.Errorf("error downloading new data: %s", err)
+	}
+
+	// refresh the file every refreshInterval
+	ticker := time.NewTicker(refreshInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			err := UpdateData(apiHost, apiKey)
+			if err != nil {
+				log.Println("Error refreshing data:", err)
+			}
+		}
+	}
+}