Remove dependency on SD/SPIFFS from CertStore (#4760)

Due to popular demand, remove the hardcoded dependency on SPIFFS
or SD from the CertStore by factoring out the file interface into
a new class (CertStoreFile) that the user will need to implement
as a thin wrapper around either a SPIFFS.file or a SD.file

Combine the downloaded certificates into a UNIX "ar" archive
and parse that on-the-fly to allow easy inspection and creation
of the Cert Store database.

Examples updated with a new certificate downloader that creates
the certs.ar archive and with a single sample that can be built
for either SPIFFS or SD with a #define.  Users can copy the
implementation of the CertStoreFile they need to their own code
as it is self-contained.

Also move the CertStore to the BearSSL namespace and remove the
suffix and separate SPIFFS/SD sources.

Remove the "deep+" change from the CI build as well (no special
options needed on any PIO or makefile build).

We'll revisit the filesystem wrapper for 2.5.0, hopefully having a
unified template for both filesystem usage at a global level.  For
current users, be aware the interface may change (simplify!) in
release 2.5.0.

Fixes #4740

Author: earlephilhower

libraries/ESP8266WiFi/examples/BearSSL_CertStore/BearSSL_CertStore.ino

@@ -7,15 +7,15 @@
 //
 // Why would you need a CertStore?
 //
-// If you know the exact serve being connected to, or you
+// If you know the exact server being connected to, or you
 // are generating your own self-signed certificates and aren't
 // allowing connections to HTTPS/TLS servers out of your
 // control, then you do NOT want a CertStore.  Hardcode the
 // self-signing CA or the site's x.509 certificate directly.
 //
 // However, if you don't know what specific sites the system
 // will be required to connect to and verify, a
-// CertStore{SPIFFS,SD}BearSSL can allow you to select from
+// CertStore can allow you to select from among
 // 10s or 100s of CAs against which you can check the
 // target's X.509, without taking any more RAM than a single
 // certificate.  This is the same way that standard browsers
@@ -31,7 +31,7 @@
 // Released to the public domain
 
 #include <ESP8266WiFi.h>
-#include <CertStoreSPIFFSBearSSL.h>
+#include <CertStoreBearSSL.h>
 #include <time.h>
 
 const char *ssid = "....";
@@ -40,7 +40,87 @@ const char *pass = "....";
 // A single, global CertStore which can be used by all
 // connections.  Needs to stay live the entire time any of
 // the WiFiClientBearSSLs are present.
-CertStoreSPIFFSBearSSL certStore;
+BearSSL::CertStore certStore;
+
+// Uncomment below to use the SD card to store the certs
+// #define USE_SDCARD 1
+
+// NOTE: The CertStoreFile virtual class may migrate to a templated
+// model in a future release. Expect some changes to the interface,
+// no matter what, as the SD and SPIFFS filesystem get unified.
+
+#ifdef USE_SDCARD
+
+#include <SD.h>
+class SDCertStoreFile : public BearSSL::CertStoreFile {
+  public:
+    SDCertStoreFile(const char *name) {
+      _name = name;
+    };
+    virtual ~SDCertStoreFile() override {};
+
+    // The main API
+    virtual bool open(bool write = false) override {
+      _file = SD.open(_name, write ? FILE_WRITE : FILE_READ);
+      return _file;
+    }
+    virtual bool seek(size_t absolute_pos) override {
+      return _file.seek(absolute_pos);
+    }
+    virtual ssize_t read(void *dest, size_t bytes) override {
+      return _file.read(dest, bytes);
+    }
+    virtual ssize_t write(void *dest, size_t bytes) override {
+      return _file.write((const uint8_t*)dest, bytes);
+    }
+    virtual void close() override {
+      _file.close();
+    }
+
+  private:
+    File _file;
+    const char *_name;
+};
+
+SDCertStoreFile certs_idx("/certs.idx");
+SDCertStoreFile certs_ar("/certs.ar");
+
+#else
+
+#include <FS.h>
+class SPIFFSCertStoreFile : public BearSSL::CertStoreFile {
+  public:
+    SPIFFSCertStoreFile(const char *name) {
+      _name = name;
+    };
+    virtual ~SPIFFSCertStoreFile() override {};
+
+    // The main API
+    virtual bool open(bool write = false) override {
+      _file = SPIFFS.open(_name, write ? "w" : "r");
+      return _file;
+    }
+    virtual bool seek(size_t absolute_pos) override {
+      return _file.seek(absolute_pos, SeekSet);
+    }
+    virtual ssize_t read(void *dest, size_t bytes) override {
+      return _file.readBytes((char*)dest, bytes);
+    }
+    virtual ssize_t write(void *dest, size_t bytes) override {
+      return _file.write((uint8_t*)dest, bytes);
+    }
+    virtual void close() override {
+      _file.close();
+    }
+
+  private:
+    File _file;
+    const char *_name;
+};
+
+SPIFFSCertStoreFile certs_idx("/certs.idx");
+SPIFFSCertStoreFile certs_ar("/certs.ar");
+#endif
 
 // Set time via NTP, as required for x.509 validation
 void setClock() {
@@ -108,6 +188,12 @@ void setup() {
   Serial.println();
   Serial.println();
 
+  #ifdef USE_SDCARD
+  SD.begin();
+  #else
+  SPIFFS.begin();
+  #endif
+
   // We start by connecting to a WiFi network
   Serial.print("Connecting to ");
   Serial.println(ssid);
@@ -126,7 +212,7 @@ void setup() {
 
   setClock(); // Required for X.509 validation
 
-  int numCerts = certStore.initCertStore();
+  int numCerts = certStore.initCertStore(&certs_idx, &certs_ar);
   Serial.printf("Number of CA certs read: %d\n", numCerts);
   if (numCerts == 0) {
     Serial.printf("No certs found. Did you run certs-from-mozill.py and upload the SPIFFS directory before running?\n");

libraries/ESP8266WiFi/examples/BearSSL_CertStore/certs-from-mozilla.py

@@ -9,8 +9,8 @@
 # Script by Earle F. Philhower, III.  Released to the public domain.
 
 import csv
-from os import mkdir
-from subprocess import Popen, PIPE
+import os
+from subprocess import Popen, PIPE, call
 import urllib2
 try:
     # for Python 2.x
@@ -40,12 +40,27 @@
 except:
     pass
 
+derFiles = []
+idx = 0
 # Process the text PEM using openssl into DER files
 for i in range(0, len(pems)):
-    certName = "data/ca_%03d.der" % (i);
+    certName = "data/ca_%03d.der" % (idx);
     thisPem = pems[i].replace("'", "")
     print names[i] + " -> " + certName
-    pipe = Popen(['openssl','x509','-inform','PEM','-outform','DER','-out', certName], shell = False, stdin = PIPE).stdin
+    ssl = Popen(['openssl','x509','-inform','PEM','-outform','DER','-out', certName], shell = False, stdin = PIPE)
+    pipe = ssl.stdin
     pipe.write(thisPem)
-    pipe.close
+    pipe.close()
+    ssl.wait()
+    if os.path.exists(certName):
+        derFiles.append(certName)
+        idx = idx + 1
 
+if os.path.exists("data/certs.ar"):
+    os.unlink("data/certs.ar");
+
+arCmd = ['ar', 'mcs', 'data/certs.ar'] + derFiles;
+call( arCmd )
+
+for der in derFiles:
+    os.unlink(der)