From 7267c1dc21883e8572f164c9564cebbf6d593dc6 Mon Sep 17 00:00:00 2001 From: Eduardo Arias Date: Wed, 15 May 2024 06:34:34 -0700 Subject: [PATCH 1/4] Added support to run regression tests without libxml2 - Annotated regression tests that depend on libxml2 support - Added Windows build without libxml2 --- .github/workflows/ci.yml | 1 + test/regression/regression.cc | 6 +++--- .../regression/action-ctl_request_body_processor.json | 3 +++ test/test-cases/regression/action-xmlns.json | 1 + test/test-cases/regression/config-body_limits.json | 2 ++ test/test-cases/regression/config-xml_external_entity.json | 3 +++ .../regression/request-body-parser-xml-validade-dtd.json | 4 ++++ test/test-cases/regression/request-body-parser-xml.json | 5 +++++ .../regression/variable-REQBODY_PROCESSOR_ERROR.json | 1 + test/test-cases/regression/variable-XML.json | 1 + 10 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34f345d696..2a870a581e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,7 @@ jobs: - {label: "full", opt: "" } - {label: "wo lmdb", opt: "-DWITHOUT_LMDB=ON" } - {label: "wo lua", opt: "-DWITHOUT_LUA=ON" } + - {label: "wo libxml2", opt: "-WITHOUT_LIBXML2=ON" } - {label: "wo maxmind", opt: "-DWITHOUT_MAXMIND=ON" } - {label: "wo curl", opt: "-DWITHOUT_CURL=ON" } steps: diff --git a/test/regression/regression.cc b/test/regression/regression.cc index 1420ddcd0f..759b7dbb82 100644 --- a/test/regression/regression.cc +++ b/test/regression/regression.cc @@ -486,15 +486,12 @@ int main(int argc, char **argv) { #if defined(WITH_GEOIP) or defined(WITH_MAXMIND) resources.push_back("geoip-or-maxmind"); #endif - #if defined(WITH_MAXMIND) resources.push_back("maxmind"); #endif - #if defined(WITH_GEOIP) resources.push_back("geoip"); #endif - #ifdef WITH_CURL resources.push_back("curl"); #endif @@ -504,6 +501,9 @@ int main(int argc, char **argv) { #ifdef WITH_LUA resources.push_back("lua"); #endif +#ifdef WITH_LIBXML2 + resources.push_back("libxml2"); +#endif #ifdef NO_LOGS std::cout << "Test utility cannot work without logging support." \ diff --git a/test/test-cases/regression/action-ctl_request_body_processor.json b/test/test-cases/regression/action-ctl_request_body_processor.json index d36a79e0fc..d2b6bd7da0 100644 --- a/test/test-cases/regression/action-ctl_request_body_processor.json +++ b/test/test-cases/regression/action-ctl_request_body_processor.json @@ -2,6 +2,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing CtlRequestBodyProcessor=XML (1)", "expected":{ "debug_log": "Registered XML namespace href \"http://schemas.xmlsoap.org/soap/envelope/\" prefix \"soap\"" @@ -71,6 +72,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing CtlRequestBodyProcessor=XML (2)", "expected":{ "debug_log": "Rule returned 0" @@ -139,6 +141,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing CtlRequestBodyProcessor=XML (3)", "expected":{ "debug_log": "XML: Failed parsing document." diff --git a/test/test-cases/regression/action-xmlns.json b/test/test-cases/regression/action-xmlns.json index f85a1d2273..df612f20c3 100644 --- a/test/test-cases/regression/action-xmlns.json +++ b/test/test-cases/regression/action-xmlns.json @@ -38,6 +38,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing XML request body parser (validate ok)", "expected":{ "debug_log": "Target value: \"39.95\" \\(Variable: XML:\/bookstore\/book\/price\\[text\\(\\)\\]\\)" diff --git a/test/test-cases/regression/config-body_limits.json b/test/test-cases/regression/config-body_limits.json index ebc047db56..c7ce75f949 100644 --- a/test/test-cases/regression/config-body_limits.json +++ b/test/test-cases/regression/config-body_limits.json @@ -578,6 +578,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"SecRequestBodyNoFilesLimit - xml, limit exceeded", "client":{ "ip":"200.249.12.31", @@ -626,6 +627,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"SecRequestBodyNoFilesLimit - xml, limit not exceeded", "client":{ "ip":"200.249.12.31", diff --git a/test/test-cases/regression/config-xml_external_entity.json b/test/test-cases/regression/config-xml_external_entity.json index fa0b405fa7..1f5cf098dc 100644 --- a/test/test-cases/regression/config-xml_external_entity.json +++ b/test/test-cases/regression/config-xml_external_entity.json @@ -2,6 +2,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing SecXMLExternalEntity/XXE 1", "expected":{ "debug_log": "Target value: \" jo smith\"" @@ -47,6 +48,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing SecXMLExternalEntity/XXE 2", "expected":{ "debug_log": "XML: Failed to load DTD: test-cases/data/SoapEnvelope.dtd", @@ -94,6 +96,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing SecXMLExternalEntity/XXE 3", "expected":{ "debug_log": "XML Error: No declaration for element bookstore", diff --git a/test/test-cases/regression/request-body-parser-xml-validade-dtd.json b/test/test-cases/regression/request-body-parser-xml-validade-dtd.json index c01c8c7522..db071aefae 100644 --- a/test/test-cases/regression/request-body-parser-xml-validade-dtd.json +++ b/test/test-cases/regression/request-body-parser-xml-validade-dtd.json @@ -2,6 +2,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing XML request body parser - validateDTD (validate ok)", "expected":{ "debug_log": "XML: Successfully validated payload against DTD: test-cases/data/SoapEnvelope.dtd" @@ -47,6 +48,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing XML request body parser - validateDTD (validation failed)", "expected":{ "debug_log": "XML Error: No declaration for element xBody", @@ -93,6 +95,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing XML request body parser - validateDTD (bad XML)", "expected":{ "debug_log": "XML: DTD validation failed because content is not well formed", @@ -139,6 +142,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing XML request body parser - validateDTD (bad DTD)", "expected":{ "debug_log": "Failed to load DTD: test-cases/data/SoapEnvelope-bad.dtd", diff --git a/test/test-cases/regression/request-body-parser-xml.json b/test/test-cases/regression/request-body-parser-xml.json index 072912d411..9b07c2ca1d 100644 --- a/test/test-cases/regression/request-body-parser-xml.json +++ b/test/test-cases/regression/request-body-parser-xml.json @@ -2,6 +2,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing XML request body parser (validate ok)", "expected":{ "debug_log": "XML: Successfully validated payload against Schema:" @@ -51,6 +52,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing XML request body parser (validate attribute value failed)", "expected":{ "debug_log": "'badval' is not a valid value of the local atomic type", @@ -101,6 +103,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing XML request body parser (validate failed)", "expected":{ "debug_log": "This element is not expected. Expected is one of", @@ -151,6 +154,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing XML request body parser (bad XML)", "expected":{ "debug_log": "XML Error: Element '{http://schemas.xmlsoap.org/soap/envelope/}xBody'", @@ -201,6 +205,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing XML request body parser (bad schema)", "expected":{ "debug_log": "XML: Failed to load Schema: test-cases/data/SoapEnvelope-bad.xsd. XML Error: Failed to parse the XML resource 'test-cases/data/SoapEnvelope-bad.xsd", diff --git a/test/test-cases/regression/variable-REQBODY_PROCESSOR_ERROR.json b/test/test-cases/regression/variable-REQBODY_PROCESSOR_ERROR.json index e470362d02..8cc4a15735 100644 --- a/test/test-cases/regression/variable-REQBODY_PROCESSOR_ERROR.json +++ b/test/test-cases/regression/variable-REQBODY_PROCESSOR_ERROR.json @@ -2,6 +2,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing Variables :: REQBODY_PROCESSOR_ERROR_MSG (1/2)", "client":{ "ip":"200.249.12.31", diff --git a/test/test-cases/regression/variable-XML.json b/test/test-cases/regression/variable-XML.json index c5ca889c19..dda3822128 100644 --- a/test/test-cases/regression/variable-XML.json +++ b/test/test-cases/regression/variable-XML.json @@ -2,6 +2,7 @@ { "enabled":1, "version_min":300000, + "resource":"libxml2", "title":"Testing XPath expression with equals sign", "expected":{ "http_code": 403 From 636cf43d5e6ba1561e76d32e20e4785e9e43d8ab Mon Sep 17 00:00:00 2001 From: Eduardo Arias Date: Sun, 5 May 2024 13:25:46 -0300 Subject: [PATCH 2/4] Separate workflow to run check-static (cppcheck) build step --- .github/workflows/ci.yml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a870a581e..5918fab9d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Setup Dependencies run: | sudo apt-get update -y -qq - sudo apt-get install -y libfuzzy-dev libyajl-dev libgeoip-dev liblua5.2-dev liblmdb-dev cppcheck libmaxminddb-dev libcurl4-openssl-dev libpcre2-dev pcre2-utils + sudo apt-get install -y libfuzzy-dev libyajl-dev libgeoip-dev liblua5.2-dev liblmdb-dev libmaxminddb-dev libcurl4-openssl-dev libpcre2-dev pcre2-utils - uses: actions/checkout@v2 with: submodules: true @@ -38,8 +38,6 @@ jobs: run: make -j `nproc` - name: check run: make check - - name: check-static - run: make check-static build-macos: runs-on: ${{ matrix.os }} @@ -125,3 +123,21 @@ jobs: working-directory: build\win32\build run: | ctest -C ${{ matrix.configuration }} --output-on-failure + + cppcheck: + runs-on: [ubuntu-22.04] + steps: + - name: Setup Dependencies + run: | + sudo apt-get update -y -qq + sudo apt-get install -y cppcheck + - name: Get libModSecurity v3 source + uses: actions/checkout@v4 + with: + submodules: true + - name: Configure libModSecurity + run: | + ./build.sh + ./configure + - name: Run cppcheck on libModSecurity + run: make check-static \ No newline at end of file From d0108efbc33fa5ec290ae7d19301924327a630ce Mon Sep 17 00:00:00 2001 From: Eduardo Arias Date: Wed, 15 May 2024 06:54:13 -0700 Subject: [PATCH 3/4] Update actions/checkout version to avoid deprecation warnings on GH workflow --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5918fab9d7..c82ecfc650 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: run: | sudo apt-get update -y -qq sudo apt-get install -y libfuzzy-dev libyajl-dev libgeoip-dev liblua5.2-dev liblmdb-dev libmaxminddb-dev libcurl4-openssl-dev libpcre2-dev pcre2-utils - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true - name: build.sh @@ -58,7 +58,7 @@ jobs: - name: Setup Dependencies run: | brew install autoconf automake cppcheck lmdb libyaml lua ssdeep libmaxminddb bison - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true - name: build.sh From d9255d85caf135b74974f2d6bbe017230a5633a1 Mon Sep 17 00:00:00 2001 From: Eduardo Arias Date: Wed, 15 May 2024 06:39:14 -0700 Subject: [PATCH 4/4] Updated GH Unix build configurations - Added support to build 32-bit versions of libModSecurity on Linux - Added support to build libModSecurity using clang on Linux (both 64-bit and 32-bit versions) - Fixed macOS dependencies to include yajl, not only because it is a required dependency, but because tests were not being run on macOS builds without it. - Added build 'without libxml' to Linux & macOS configurations. - Added build 'without ssdeep' to Linux configurations (already in macOS configuration) - Added build 'with lmdb' to Linux & macOS configurations, replacing the existing one 'without lmdb' because by default LMDB is disabled if not explicitly turn on in configure. - Removed 'without yajl' build because it's a required 3rd party dependency. - Added bison & flex dependencies to enable parser generation. --- .github/workflows/ci.yml | 96 ++++++++++++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c82ecfc650..b081fb93bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,33 +6,66 @@ on: jobs: build-linux: + name: Linux (${{ matrix.platform.label }}, ${{ matrix.compiler.label }}, ${{ matrix.configure.label }}) runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-22.04] - platform: [x32, x64] - compiler: [gcc, clang] + platform: + - {label: "x64", arch: "amd64", configure: ""} + - {label: "x32", arch: "i386", configure: "PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32"} + compiler: + - {label: "gcc", cc: "gcc", cxx: "g++"} + - {label: "clang", cc: "clang", cxx: "clang++"} configure: - {label: "with parser generation", opt: "--enable-parser-generation" } - {label: "wo curl", opt: "--without-curl" } - - {label: "wo yajl", opt: "--without-yajl" } - - {label: "wo geoip", opt: "--without-geoip" } - - {label: "wo lmdb", opt: "--without-lmdb" } - - {label: "with pcre2", opt: "--with-pcre2" } - {label: "wo lua", opt: "--without-lua" } - - {label: "without maxmind", opt: "--without-maxmind" } + - {label: "wo maxmind", opt: "--without-maxmind" } + - {label: "wo libxml", opt: "--without-libxml" } + - {label: "wo geoip", opt: "--without-geoip" } + - {label: "wo ssdeep", opt: "--without-ssdeep" } + - {label: "with lmdb", opt: "--with-lmdb" } + - {label: "with pcre2", opt: "--with-pcre2" } + exclude: + - platform: {label: "x32"} + configure: {label: "wo geoip"} + - platform: {label: "x32"} + configure: {label: "wo ssdeep"} steps: - - name: Setup Dependencies + - name: Setup Dependencies (common) run: | + sudo dpkg --add-architecture ${{ matrix.platform.arch }} sudo apt-get update -y -qq - sudo apt-get install -y libfuzzy-dev libyajl-dev libgeoip-dev liblua5.2-dev liblmdb-dev libmaxminddb-dev libcurl4-openssl-dev libpcre2-dev pcre2-utils + sudo apt-get install -y libyajl-dev:${{ matrix.platform.arch }} \ + libcurl4-openssl-dev:${{ matrix.platform.arch }} \ + liblmdb-dev:${{ matrix.platform.arch }} \ + liblua5.2-dev:${{ matrix.platform.arch }} \ + libmaxminddb-dev:${{ matrix.platform.arch }} \ + libpcre2-dev:${{ matrix.platform.arch }} \ + pcre2-utils:${{ matrix.platform.arch }} \ + bison flex + - name: Setup Dependencies (x32) + if: ${{ matrix.platform.label == 'x32' }} + run: | + sudo apt-get install g++-multilib + sudo apt-get install -y libxml2-dev:${{ matrix.platform.arch }} \ + libpcre3-dev:${{ matrix.platform.arch }} + - name: Setup Dependencies (x64) + if: ${{ matrix.platform.label == 'x64' }} + run: | + sudo apt-get install -y libgeoip-dev:${{ matrix.platform.arch }} \ + libfuzzy-dev:${{ matrix.platform.arch }} - uses: actions/checkout@v4 with: submodules: true - name: build.sh run: ./build.sh - - name: configure ${{ matrix.configure.label }} - run: ./configure ${{ matrix.configure.opt }} + - name: configure + env: + CC: ${{ matrix.compiler.cc }} + CXX: ${{ matrix.compiler.cxx }} + run: ./configure ${{ matrix.platform.configure }} ${{ matrix.configure.opt }} - uses: ammaraskar/gcc-problem-matcher@master - name: make run: make -j `nproc` @@ -40,30 +73,43 @@ jobs: run: make check build-macos: + name: macOS (${{ matrix.configure.label }}) runs-on: ${{ matrix.os }} strategy: matrix: os: [macos-12] - compiler: [clang] configure: - {label: "with parser generation", opt: "--enable-parser-generation" } - {label: "wo curl", opt: "--without-curl" } - - {label: "wo yajl", opt: "--without-yajl" } - - {label: "wo geoip", opt: "--without-geoip" } - - {label: "wo lmdb", opt: "--without-lmdb" } - - {label: "wo ssdeep", opt: "--without-ssdeep" } - {label: "wo lua", opt: "--without-lua" } - {label: "wo maxmind", opt: "--without-maxmind" } + - {label: "wo libxml", opt: "--without-libxml" } + - {label: "wo geoip", opt: "--without-geoip" } + - {label: "wo ssdeep", opt: "--without-ssdeep" } + - {label: "with lmdb", opt: "--with-lmdb" } + - {label: "with pcre2", opt: "--with-pcre2" } steps: - name: Setup Dependencies + # autoconf, curl, pcre2 not installed because they're already + # included in the image run: | - brew install autoconf automake cppcheck lmdb libyaml lua ssdeep libmaxminddb bison + brew install automake \ + yajl \ + lmdb \ + lua \ + libmaxminddb \ + libxml2 \ + geoip \ + ssdeep \ + pcre \ + bison \ + flex - uses: actions/checkout@v4 with: submodules: true - name: build.sh run: ./build.sh - - name: configure ${{ matrix.configure.label }} + - name: configure run: ./configure ${{ matrix.configure.opt }} - uses: ammaraskar/gcc-problem-matcher@master - name: make @@ -72,19 +118,21 @@ jobs: run: make check build-windows: + name: Windows (${{ matrix.platform.label }}, ${{ matrix.configure.label }}) runs-on: ${{ matrix.os }} strategy: matrix: os: [windows-2022] - platform: [x86_64] + platform: + - {label: "x64", arch: "x86_64"} configuration: [Release] configure: - {label: "full", opt: "" } + - {label: "wo curl", opt: "-DWITHOUT_CURL=ON" } - {label: "wo lmdb", opt: "-DWITHOUT_LMDB=ON" } - {label: "wo lua", opt: "-DWITHOUT_LUA=ON" } - - {label: "wo libxml2", opt: "-WITHOUT_LIBXML2=ON" } - {label: "wo maxmind", opt: "-DWITHOUT_MAXMIND=ON" } - - {label: "wo curl", opt: "-DWITHOUT_CURL=ON" } + - {label: "wo libxml", opt: "-WITHOUT_LIBXML2=ON" } steps: - uses: actions/checkout@v4 with: @@ -94,9 +142,9 @@ jobs: pip3 install conan --upgrade conan profile detect - uses: ammaraskar/msvc-problem-matcher@master - - name: Build ${{ matrix.configuration }} ${{ matrix.platform }} ${{ matrix.configure.label }} + - name: Build ${{ matrix.configuration }} ${{ matrix.platform.arch }} ${{ matrix.configure.label }} shell: cmd - run: vcbuild.bat ${{ matrix.configuration }} ${{ matrix.platform }} NO_ASAN "${{ matrix.configure.opt }}" + run: vcbuild.bat ${{ matrix.configuration }} ${{ matrix.platform.arch }} NO_ASAN "${{ matrix.configure.opt }}" - name: Set up test environment working-directory: build\win32\build\${{ matrix.configuration }} env: @@ -140,4 +188,4 @@ jobs: ./build.sh ./configure - name: Run cppcheck on libModSecurity - run: make check-static \ No newline at end of file + run: make check-static