Make ArduinoOTA MKR1400/1500 compatible - Feature request

[CptHolzschnauz]

At the moment, the lib is not compatible because MKRGSM reads out the connection in 512 bytes portions, so at the end when there is just a rest of bytes, it misses the last portion.

The core is:

byte b;
  while (length > 0) {
    if (!client.readBytes(&b, 1)) // reading a byte with timeout
      break;
    file.write(b);
    length--;
  }

The Arduino download example under
https://github.com/arduino-libraries/MKRGSM/blob/master/examples/Tools/FileUtilsHttpDownload/FileUtilsHttpDownload.ino
to store the file in the memory of the modem calculates this way:

constexpr uint32_t len { 512 };
  uint32_t cycles = fileSize / len;
  uint32_t spares = fileSize % len;

fileSize is the size of the file to be downloaded

The core is:

auto downloadAndSaveTrunk = [filename_server](uint32_t len) {
    char buf[len] { 0 };
    uint32_t written { 0 };

    if (net.available())
      written = net.readBytes(buf, len);
    fileUtils.appendFile(filename_server, buf, written);
    return written;
  };

  // Define wrapper function
  auto saveTrunk = [&totalRead, downloadAndSaveTrunk](size_t iter, uint32_t len) {
    Serial.print("Download Block ");
    Serial.print(iter);
    totalRead += downloadAndSaveTrunk(len);
    Serial.print(": Size:");
    Serial.print(len);
    Serial.print(" - Total: ");
    Serial.print(totalRead);
    Serial.println(" bytes");
  };
  // Download and save the rest of the block
  for (auto c = 0; c <= cycles; c++)
    saveTrunk(c, len);
}

I tried but failed on merge this concepts (but i'm surely not skilled as JAndrassy or many others) but maybe someone can contribute a ArduinoOTA SD version to this lib?
@JAndrassy what do you think?

[JAndrassy]

I don't understand. my example uses the Arduino API of the Stream class which should work the same way for all implementations of Stream

[CptHolzschnauz]

Me neither. The MKRGSM lib does read the tcp connection with looping the AT command
AT+USORD=0,512
Meaning read out 512 bytes on tcp connection number 0.

At the end of the file, when there is not a full portion of 512 bytes avaiable, the modem struggles and answer with the very detailed report of "ERROR"..

So the file is not completely downloaded (between 1 and 512 bytes are missed) and your lib handles the error and stop the process.

The referenced Arduino example lib handles this task in the way as described and then it works.

[CptHolzschnauz]

Here are two example of the log.
The thing i dont understand is that IMHO the correct amount of bytes were delivered at the end (see example 1: 08:35:01.125 -> +USORD: 1,210,) therfore they should be read. Why are bytes missing at the end?

Always at the end the modem closes the socket and some bytes were missing:
Example 1:

8:35:01.054 -> +UUSORD: 1,722
08:35:01.054 -> AT+USORD=1,512

08:35:01.054 -> +USORD: 1,512,"00000000A0420000A0420000A0420000A042000048C2000048C2000048C2000048C20000C84210270000F04902000024744800002042439C03004B9C0300559C03005D9C0300689C0300729C03005000000039990300FFFFFFFF38A2030027A203000CA203003C000000002000420024004200280042002C0042003000420034004240420F000A0000000A00000008000000FFFFFFFFFFFFFFFF00366E01FFFFFFFF00C2010000000800FFFFFFFF010000008401002000000000C8C20300E8C20300A8C2030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004300000000000000000000000000000000000000000000000000000000000000430000000000000000000000000000000000000000000000000000000000000043000000000000000000000000000000000000000000000000000000000000004300000000000000000000000000000000000000000000000000000000000000430000000000000000000000000000000000000000000000000000000000000043000000000000000000000000000000000000000000000000000000000000004300000000000000000000000000000000000000000000000000000000000000B99E0200D98302000000"
08:35:01.125 ->
08:35:01.125 -> OK
08:35:01.125 ->
08:35:01.125 -> +UUSORD: 1,210
08:35:01.125 -> AT+USORD=1,512

08:35:01.125 -> +USORD: 1,210,"000098C10300DABD02007EC002007EC002007EC002007EC002007EC002007EC002007EC002007EC002007EC00200FFFFFFFFFFFFFFFFFFFFFFFFFFFF00000100415343494900000000000000000000000000000000000000000000000000000041534349490000000000000000000000000000000000000000000000000000000000DD600000BD9300008D3D0100FD3F010091480100514C010049520100E952010089800100B18201007D890100ADFD01003D0002005D17020071250200452C0200B5600000000000000000000000000000"
08:35:01.159 ->
08:35:01.159 -> OK
08:35:01.194 -> AT+USORD=1,512

08:35:01.409 -> ERROR
08:35:01.444 ->
08:35:01.444 -> +UUSOCL: 1
08:35:01.444 -> AT+USOCL=1

08:35:01.444 -> ERROR
08:35:02.168 -> Local checksum:19896328
08:35:02.768 -> Timeout downloading update, missing 146 bytes. Can't continue with update.
08:35:02.902 -> AT+USORD=0,512

08:35:02.902 -> +USORD: 0,0,""
08:35:02.902 ->
08:35:02.902 -> OK
08:35:02.902 -> AT+USORD=0,512

Example 2:

8:38:13.075 -> +USORD: 1,512,"00000000A0420000A0420000A0420000A042000048C2000048C2000048C2000048C20000C84210270000F04902000024744800002042439C03004B9C0300559C03005D9C0300689C0300729C03005000000039990300FFFFFFFF38A2030027A203000CA203003C000000002000420024004200280042002C0042003000420034004240420F000A0000000A00000008000000FFFFFFFFFFFFFFFF00366E01FFFFFFFF00C2010000000800FFFFFFFF010000008401002000000000C8C20300E8C20300A8C2030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004300000000000000000000000000000000000000000000000000000000000000430000000000000000000000000000000000000000000000000000000000000043000000000000000000000000000000000000000000000000000000000000004300000000000000000000000000000000000000000000000000000000000000430000000000000000000000000000000000000000000000000000000000000043000000000000000000000000000000000000000000000000000000000000004300000000000000000000000000000000000000000000000000000000000000B99E0200D98302000000"
08:38:13.147 ->
08:38:13.147 -> OK
08:38:13.147 ->
08:38:13.147 -> +UUSORD: 1,210
08:38:13.147 -> AT+USORD=1,512

08:38:13.147 -> +USORD: 1,210,"000098C10300DABD02007EC002007EC002007EC002007EC002007EC002007EC002007EC002007EC002007EC00200FFFFFFFFFFFFFFFFFFFFFFFFFFFF00000100415343494900000000000000000000000000000000000000000000000000000041534349490000000000000000000000000000000000000000000000000000000000DD600000BD9300008D3D0100FD3F010091480100514C010049520100E952010089800100B18201007D890100ADFD01003D0002005D17020071250200452C0200B5600000000000000000000000000000"
08:38:13.181 ->
08:38:13.181 -> OK
08:38:13.181 -> AT+USORD=1,512

08:38:13.443 -> ERROR
08:38:13.443 ->
08:38:13.443 -> +UUSOCL: 1
08:38:13.443 -> AT+USOCL=1

08:38:13.443 -> ERROR
08:38:14.179 -> Local checksum:19899162
08:38:14.778 -> Timeout downloading update, missing 110 bytes. Can't continue with update.
08:38:14.923 -> AT+USORD=0,512

08:38:14.923 -> +USORD: 0,0,""

[CptHolzschnauz]

I digged deeper and i found the root cause: The modem is simply not delivering the right data. I made a hexdump diff of the file on the server and with the file downloaded and the missed bytes were all over the file, not just at the end.
Conclusion:

• Nothing to do with this ArduinoOTA
• No idea what to do. Some error checks? How is this possible?

[JAndrassy]

with the file downloaded and the missed bytes were all over the file, not just at the end

this is usually caused by buffer overflows of the Serial buffer.
the flash writing takes time so the Serial buffer isn't read fast enough, it fills up a the following bytes are dropped.
adding a buffer in the read loop could help.

see how it is done in WiFiOTA.cpp

ArduinoOTA/src/WiFiOTA.cpp

Line 296 in 28a22f0

long read = 0;

[CptHolzschnauz]

I tried yesterday a solution (but with 512 byte pakets to write in one rush to the SD, i assumed the bottleneck was the SD card) with no success, i will try again following your provided example. Thanks a bunch

[CptHolzschnauz]

I implemented with no success. I also tried to reduce the #define GSM_SOCKET_BUFFER_SIZE 512 down to 16 (recommended by the modem manufacturer if the FIFO rx buffer from the serial connected device is small.
Same result: Bytes missing all over the file.

[CptHolzschnauz]

I invested another day on investigation. Conclusion remains the same - the only way to get a bin complete to a MKR1400 is into the internal memory and then from there to the SD card, for large sketches i still see only my way:
https://github.com/CptHolzschnauz/Arduino-MKR-1400-OTA-Update

[JAndrassy]

why to SD card?

[CptHolzschnauz]

Because the UPDATE.bin is around 230kB. I see no other way then update over SDU.h ?!?

[JAndrassy]

so now I wonder why you have so large binary? maybe move some large data from the binary into a file on the SD card

[CptHolzschnauz]

My whole sketch is that large. The OTA is for a complete firmware update. Your sketch works like a charm on MKR1100 over Wifi within seconds thats why i tried to have the same code on MKR1400 - without success.

[CptHolzschnauz]

I started a new attempt to bring this to work on a MKR1400 with SD card for large sketches.
The problem is again that the file saved on the SD card diffs slightly from the file on the server.
Now i found out that only at the beginning and at the end is a hex diff while all the main middle part is identical.
Example: Right the original file, left the downloaded
Start: I marked the +- start point where it starts to be identical

End: At the end some bytes are additional

I'm confused.
If the buffer is not ready as you mentioned i would expect nothing but there are bytes (from where?)
At the end, there are bytes added but the buffer is empty (in total there are missing bytes)
Any idea?

[CptHolzschnauz]

I found out that MKR 1400 ArduinoHttpClient and direct write to the SD card is not working, probably, as @JAndrassy mentioned, because of buffer time lag, whatever, it's working not properly but flaky.

This has nothing to do with the ArduinoOTA lib, so just for documentation if someone with the same problem finds this thread, a version that works with direct client.read and .print from MKRGSM. The HTTP Head removal is a hack, this one is for a Apache Header on a Ubuntu, maybe needs changes :

`#include<SDU.h>
#include <MKRGSM.h>
#include <SPI.h>
#include <SD.h>

const char PINNUMBER[] = "";
// APN data
const char GPRS_APN[] = "";
const char GPRS_LOGIN[] = "";
const char GPRS_PASSWORD[] = "";


GSMClient client;
GPRS gprs;
GSM gsmAccess;

File myFile;

char server[] = "server.ch";
char path[] = "/update-file-server.bin";
int port = 80;
bool header = true;
int nl_count;
String currentLine;

void setup() {
  // initialize serial communications and wait for port to open:
  Serial.begin(2000);
  while (!Serial) {
    ; // wait for serial port to connect. Needed for native USB port only
  }

  Serial.print("Initializing SD card...");
  if (!SD.begin(4)) {
    Serial.println("Initialization failed!");
    while (1);
  }
  Serial.println("Initialization done.");

  Serial.println("Starting Arduino web client.");
  bool connected = false;
  while (!connected) {
    if ((gsmAccess.begin(PINNUMBER) == GSM_READY) &&
        (gprs.attachGPRS(GPRS_APN, GPRS_LOGIN, GPRS_PASSWORD) == GPRS_READY)) {
      connected = true;
    } else {
      Serial.println("Not connected");
      delay(1000);
    }
  }
  Serial.println("connecting...");

  if (client.connect(server, port)) {
    Serial.println("connected");
    // Make a HTTP request:
    client.print("GET ");
    client.print(path);
    client.println(" HTTP/1.1");
    client.print("Host: ");
    client.println(server);
    client.println("Connection: close");
    client.println();
  } else {
    // if you didn't get a connection to the server:
    Serial.println("connection failed");
  }

  if (SD.exists("UPDATE.bin")) {
    Serial.println("UPDATE.bin exists.");
    // delete the file:
    Serial.println("Removing UPDATE.bin...");
    SD.remove("UPDATE.bin");
  }
  else {
    Serial.println("UPDATE.bin doesn't exist on the SD card. Open a new file..");
  }
  myFile = SD.open("UPDATE.bin", FILE_WRITE);
}
void loop()
{

  if (client.available()) {
    char c = client.read();

    if (header == true) {
      currentLine += c;
      if (c == '\n') {
        currentLine = "";
        nl_count++;
        Serial.print("NL detected:");
        Serial.print(nl_count);
        if (nl_count == 10)
        {
          Serial.print("10 NLs detected, end of Header , three bytes to digest until the payload");
          Serial.print(client.read());
          Serial.print(client.read());
          Serial.print(client.read());
          header = false;
        }
      }
    }
    if (header == false)
    {
      myFile.print(c);
    }
    //Serial.print(c);
  }

  // if the server's disconnected, stop the client:
  if (!client.available() && !client.connected()) {
    Serial.println();
    Serial.println("disconnecting.");
    client.stop();
    myFile.close();
    Serial.println("done.");
    // do nothing forevermore:
    for (;;)
      ;
  }
}

[CptHolzschnauz]

Final conclusion:
I had a WDTZero watchdog in place. With soft WDT function, every time-slot the EW(early warning) IRQ is issued to update the soft counter. This takes some processor time and data is lost randomly.