Skip to content

Commit

Permalink
Merge branch 'v5.4' of github.com:e2guardian/e2guardian into v5.4
Browse files Browse the repository at this point in the history
  • Loading branch information
philipianpearce committed Nov 23, 2021
2 parents eae46a7 + efed3eb commit d8da3f3
Show file tree
Hide file tree
Showing 16 changed files with 195 additions and 93 deletions.
28 changes: 21 additions & 7 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,35 @@ cache:

stages:
# At first debian : init variables
- quality
- build-debian
- create-debian-package
- create-package-debian
- test-publish-debian
- Docker-hub-build
- Docker-hub-test
- Docker-hub-pushtag

### All
- build
- create-package
- test-publish
# AMD64 docker
- Docker-hub-build
- Docker-hub-test
- Docker-hub-pushtag
# ARM docker
- Docker-hub-build-arm
- Docker-hub-test-arm
- Docker-hub-pushtag-arm

hadolint:
image: hadolint/hadolint:latest-debian
stage: quality
before_script:
- cd $CI_PROJECT_DIR
script:
- hadolint --ignore DL3008 gitlabci/docker-ci/Dockerfile

include:
- 'gitlabci/docker-hub-arm.yml'
- 'gitlabci/docker-hub.yml'
- 'gitlabci/debianbuster.yml'
- 'gitlabci/debianlatest.yml'
- 'gitlabci/ubuntubionic.yml'
- 'gitlabci/raspbianbuster.yml'
- 'gitlabci/armdebian.yml'
- 'gitlabci/ubuntufocal.yml'
8 changes: 4 additions & 4 deletions configs/e2guardian.conf.in
Original file line number Diff line number Diff line change
Expand Up @@ -735,7 +735,7 @@ reverseaddresslookups = off
# and normally e2g will no longer check site lists for ip's
# If you want to keep backward list compatablity then set this to
# 'on' - but note this incurs an overhead - putting IP in ipsitelists
# and setting this to off gives the fastest implimentation.
# and setting this to off gives the fastest implementation.
# default is 'on'

###
Expand Down Expand Up @@ -1033,7 +1033,7 @@ weightedphrasemode = 2
# The size is in Kibibytes - eg 2048 = 2Mb
# default 2048

#maxcontentramcachescansize = 2000
#maxcontentramcachescansize = 2048
#
# Max content ram cache scan size
# This is only used if you use a content scanner plugin such as AV
Expand All @@ -1043,7 +1043,7 @@ weightedphrasemode = 2
# The size is in Kibibytes - eg 10240 = 10Mb
# use 0 to set it to maxcontentfilecachescansize
# This option may be ignored by the configured download manager.
# default 2000
# default 2048

#maxcontentfilecachescansize = 20000
#
Expand Down Expand Up @@ -1140,7 +1140,7 @@ weightedphrasemode = 2
## Things that will only work if specifically compliled

## 'NEW' debug system (generaly compliled in release systems)
## Note that this is only partialy implimented and only works for ICAP, CLAMAV
## Note that this is only partialy implemented and only works for ICAP, CLAMAV
## and ICAPC
## and so 'ALL' = 'ICAP,CLAMAV,ICAPC' only.
## To debug other areas of code re-compiling in debug mode is required
Expand Down
8 changes: 4 additions & 4 deletions configs/e2guardianf1.conf.in
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
### URL_MOD - Lists to modify url and ssl site target
### REDIRECT - Redirection list
### LOG_ONLY - Categorise and log but do not filter lists
### POST - Post filtering - not implimented yet in v5
### POST - Post filtering - not implemented yet in v5
### PHRASES - Content phrase settings and lists
### SEARCH - Search term filtering settings and lists
### AV_SCANNERS - AV scanner settings and lists
Expand Down Expand Up @@ -44,7 +44,7 @@
### Copy examplef1.story to group1.story
### In list directory:-
### Create a group1 directory
### Copy the lists in lists/sample.group into this directory
### Copy the lists in lists/example.group into this directory
### and edit them to your requirements
### Comment out .define.., and storyboard = lines below.
###
Expand Down Expand Up @@ -809,7 +809,7 @@ sitelist = 'name=bannedbypass,messageno=500,path=__LISTDIR__/domainsnobypass'
# Enable legacy (E2) ssl logic
# The option is replaced by storyboard logic

#sslcertcheck = off - NOT implimented in V5 yet
#sslcertcheck = off - NOT implemented in V5 yet
#SSL certificate checking
# Check that ssl certificates for servers on https connections are valid
# and signed by a ca in the configured path
Expand All @@ -822,7 +822,7 @@ sitelist = 'name=bannedbypass,messageno=500,path=__LISTDIR__/domainsnobypass'
# option is replaced by storyboard logic

#embeddedurlweight = 0
# - NOT implimented in v5
# - NOT implemented in v5
# Embedded URL weighting
# When set to something greater than zero, this option causes URLs embedded within a
# page's HTML (from links, image tags, etc.) to be extracted and checked against the
Expand Down
2 changes: 1 addition & 1 deletion configs/lists/example.group/bannedsitelist.in
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@

# badboys.com

# NOTE: From v5 Blanket blocks are now implimented using Storyboarding
# NOTE: From v5 Blanket blocks are now implemented using Storyboarding
# WARNING: Old style Blanket blocks in this file will be silently ignored


Expand Down
2 changes: 1 addition & 1 deletion configs/lists/example.group/exceptionsitelist
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
# Remove the first # from the line above to enable this list only from
# 9am to 5pm, Monday to Friday.

# NOTE: From v5 Blanket exceptions are now implimented using Storyboarding
# NOTE: From v5 Blanket exceptions are now implemented using Storyboarding
# WARNING: Old style Blanket blocks in this file will be silently ignored

windowsupdate.microsoft.com
1 change: 0 additions & 1 deletion configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ AC_DEFINE([__SSLMITM],[""],[Define to enable SSL MITM])

AC_PREREQ(2.57)
AC_INIT(e2guardian, 5.4.4)
AM_INIT_AUTOMAKE
AC_CONFIG_HEADERS([e2config.h])
AC_CONFIG_MACRO_DIR([m4])
AM_INIT_AUTOMAKE([subdir-objects])
Expand Down
65 changes: 50 additions & 15 deletions gitlabci/raspbianbuster.yml → gitlabci/armdebian.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
build:raspbian:
build:arm:
stage: build
image: amd64/debian:stable
image: debian:bullseye
artifacts:
expire_in: 30 minutes
paths:
Expand All @@ -12,31 +12,28 @@ build:raspbian:
- echo $VFULL
- apt update
- apt-get -y upgrade
- apt install --no-install-recommends --no-install-suggests -y curl unzip base-files automake base-passwd pkg-config bash coreutils debianutils diffutils dpkg e2fsprogs findutils grep gzip hostname ncurses-base tar git bsdutils lsb-release
- echo "deb-src http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list
- dpkg --add-architecture armhf
- apt-get update
- apt-get install -y build-essential crossbuild-essential-armhf
- apt-get build-dep -y -a armhf e2guardian
- echo "deb-src http://deb.debian.org/debian bullseye main contrib non-free" >> /etc/apt/sources.list
- apt-get install --no-upgrade --no-install-recommends --no-install-suggests -y curl unzip automake coreutils debianutils diffutils e2fsprogs findutils grep unzip ncurses-base libevent-pthreads-* libevent-dev ncurses-bin login sysvinit-utils tar libc6-dev libc-dev gcc g++ make dpkg-dev autotools-dev debhelper dh-autoreconf dpatch libclamav-dev libpcre3-dev zlib1g-dev pkg-config libssl-dev libssl1.1 ca-certificates lsb-release inotify-tools curl
- cd $CI_PROJECT_DIR && make clean
- ./autogen.sh
- ./configure --host=arm-linux-gnueabihf --prefix=/usr --enable-clamd=yes --with-proxyuser=e2guardian --with-proxygroup=e2guardian --sysconfdir=/etc --localstatedir=/var --enable-icap=yes --enable-commandline=yes --enable-email=yes --enable-ntlm=yes --enable-pcre=yes --enable-sslmitm=yes
- make ARCH=arm-linux-gnueabihf
- file src/e2guardian | grep "ARM, EABI5"
- make
- find $CI_PROJECT_DIR -name ".git" -exec rm -r "{}" +
tags:
- arm

package:raspbian:
package:debarm:
stage: create-package
image: amd64/debian:stable
image: debian:bullseye
dependencies:
- build:raspbian
- build:arm
artifacts:
paths:
- $CI_PROJECT_DIR
variables:
PACKPATH: "$CI_PROJECT_DIR"
PACKDST: "$CI_PROJECT_DIR/scripts/debian_package/e2raspbian_package"
OS: "raspbian"
PACKDST: "$CI_PROJECT_DIR/scripts/debian_package/e2debian-arm64_package"
OS: "debian-arm64"
script:
- source $CI_PROJECT_DIR/scripts/debian_package/variables
- apt update
Expand All @@ -56,6 +53,42 @@ package:raspbian:
- find ${PACKDST}/ -type f -name "Makefil*" -delete
- find ${PACKDST}/ -type f -name "*.in" -delete
- cd ${PACKDST} && cd .. && ./rebuild.sh e2"$OS"_package
- find $CI_PROJECT_DIR -name ".git" -exec rm -r "{}" +
tags:
- arm

test:debianarm:
stage: test-publish
image: debian:bullseye
dependencies:
- package:debarm
artifacts:
name: builddebian
paths:
- $CI_PROJECT_DIR
variables:
PACKPATH: "$CI_PROJECT_DIR"
PACKDST: "$CI_PROJECT_DIR/scripts/debian_package/e2debian-arm64_package"
CERT_DIR: "/etc/e2guardian/private/"
OS: "debian-arm64"
script:
- source $CI_PROJECT_DIR/scripts/debian_package/variables
- echo $VFULL
- apt update
- apt install -y openssl curl git lsb-release libevent-pthreads* libtommath*
- VERSION=`lsb_release -cs`
- cd ${PACKDST} && cd .. && dpkg -i e2"$OS"_package.deb
- sed -i "s/^#sslmitm.*$/sslmitm\ =\ on/" /etc/e2guardian/e2guardianf1.conf && sed -i "s/^#enablessl.*$/enablessl\ =\ on/" /etc/e2guardian/e2guardian.conf && sed -i "s/^RANDFILE\s*=\s*\\\$ENV/#RANDFILE\ =\ \$ENV/" /etc/ssl/openssl.cnf
- mkdir -p ${CERT_DIR}generatedcerts
- openssl genrsa 4096 > "${CERT_DIR}ca.key" && openssl req -subj "/C=US" -new -x509 -days 3650 -key "${CERT_DIR}ca.key" -out "${CERT_DIR}ca.pem" && openssl genrsa 4096 > "${CERT_DIR}cert.key"
- chown --recursive e2guardian:e2guardian ${CERT_DIR}
- echo "qwant.com" >> /etc/e2guardian/lists/example.group/bannedsitelist
- e2guardian -N &
- sleep 5 && export https_proxy=http://localhost:8080
- curl -k -o /tmp/test https://www.google.com
- curl -k https://www.qwant.com | grep -i e2guardian
- unset https_proxy
- mkdir -p "${CI_COMMIT_BRANCH}"
- mv e2"$OS"_package.deb ${CI_COMMIT_BRANCH}/e2"$OS"_"$VERSION"_V"$VFULL"_$(date +%Y%m%d).deb
- md5sum ${CI_COMMIT_BRANCH}/e2"$OS"_"$VERSION"_V"$VFULL"_$(date +%Y%m%d).deb > ${CI_COMMIT_BRANCH}/e2"$OS"_"$VERSION"_V"$VFULL"_$(date +%Y%m%d)_md5.txt
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
Expand All @@ -65,3 +98,5 @@ package:raspbian:
- ssh-add <(echo "$SSH_NOSTROMO_KEY")
- scp -P 822 -r ${CI_COMMIT_BRANCH} [email protected]:/datas/e2/html
- find $CI_PROJECT_DIR -name ".git" -exec rm -r "{}" +
tags:
- arm
14 changes: 7 additions & 7 deletions gitlabci/debianbuster.yml → gitlabci/debianlatest.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
Build debian package:
build:debian:
stage: build-debian
artifacts:
expire_in: 30 minutes
paths:
- $CI_PROJECT_DIR
image: amd64/debian:stable
image: amd64/debian:bullseye
variables:
OS: "debian"
script:
Expand All @@ -19,9 +19,9 @@ Build debian package:
- make
- find $CI_PROJECT_DIR -name ".git" -exec rm -r "{}" +

Create Debian Package:
stage: create-debian-package
image: amd64/debian:stable
package:debian:
stage: create-package-debian
image: amd64/debian:bullseye
artifacts:
paths:
- $CI_PROJECT_DIR
Expand All @@ -48,9 +48,9 @@ Create Debian Package:
- cd ${PACKDST} && cd .. && ./rebuild.sh e2"$OS"_package
- find $CI_PROJECT_DIR -name ".git" -exec rm -r "{}" +

Test and publish Debian:
test:debian:
stage: test-publish-debian
image: amd64/debian:stable
image: amd64/debian:bullseye
artifacts:
name: builddebian
paths:
Expand Down
37 changes: 19 additions & 18 deletions gitlabci/docker-ci/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,28 +1,29 @@
FROM debian:buster

LABEL commit.e2guardian=$COMMIT build_date.e2guardian=$BUILD_DATE
FROM debian:bullseye-slim as buildamd
COPY . /tmp/e2guardian
RUN cd /tmp/e2guardian \
&& apt-get update && apt-get install --no-install-recommends --no-install-suggests -y sed libevent-pthreads-* libssl1.1 ca-certificates \
WORKDIR /tmp/e2guardian

FROM debian:bullseye-slim
WORKDIR /tmp/e2guardian
ENV DOCKER_BUILD=/tmp/e2guardian/scripts/debian_package/e2debian_package/data
COPY --from=buildamd --chown=1161 /tmp/e2guardian/src/e2guardian /usr/sbin/e2guardian
COPY --from=buildamd --chown=1161 $DOCKER_BUILD/etc/e2guardian /etc/e2guardian
COPY --from=buildamd --chown=1161 $DOCKER_BUILD/usr/share/e2guardian/languages /usr/share/e2guardian/languages
COPY --from=buildamd --chown=1161 $DOCKER_BUILD/usr/share/e2guardian/*swf /usr/share/e2guardian/
COPY --from=buildamd --chown=1161 $DOCKER_BUILD/usr/share/e2guardian/*gif /usr/share/e2guardian/
RUN apt-get update && apt-get install --no-upgrade --no-install-recommends --no-install-suggests -y sed libevent-pthreads-* libssl1.1 ca-certificates curl \
&& adduser --no-create-home --uid 1161 --group --system e2guardian \
&& cp src/e2guardian /usr/sbin/e2guardian \
&& chmod +x /usr/sbin/e2guardian \
&& cp -Rf configs /etc/e2guardian \
&& mkdir -p /usr/share/e2guardian/languages \
&& mkdir -p /run/e2guardian \
&& mkdir -p /var/log/e2guardian && chown -R e2guardian /var/log/e2guardian && chown -R e2guardian /etc/e2guardian && chown -R e2guardian /run/e2guardian \
&& cp -Rf data/languages /usr/share/e2guardian/ \
&& cp data/*.gif /usr/share/e2guardian/ \
&& cp data/*swf /usr/share/e2guardian/ \
&& mkdir -p /var/log/e2guardian && chown -R e2guardian /var/log/e2guardian && chown -R e2guardian /run/e2guardian \
&& rm -Rf /tmp/* \
&& sed -i "s/^#dockermode.*$/dockermode\ =\ on/" /etc/e2guardian/e2guardian.conf \
&& sed -i "s/^#pidfilename.*$/pidfilename\ =\ \/run\/e2guardian\/e2.pid/" /etc/e2guardian/e2guardian.conf \
&& sed -i "s/^#icapport.*$/icapport\ =\ 1344/" /etc/e2guardian/e2guardian.conf \
&& apt-get autoremove -y \
&& rm -rf /var/lib/apt/lists/* && rm -Rf /tmp/*
&& apt-get clean \
&& rm -Rf /var/lib/apt/lists/*

COPY --chown=e2guardian:root run.sh /
RUN chmod +x /run.sh
HEALTHCHECK CMD curl localhost:8080 || exit 1
HEALTHCHECK CMD curl --proxy "http://127.0.0.1:8080" http://internal.test.e2guardian.org | grep "e2guardian internal test OK" || exit 1
USER e2guardian
EXPOSE 8080
ENTRYPOINT ["/run.sh"]
#Shell will take up PID 1, and e2guardian will be its child proces
CMD ["/bin/sh", "-c","e2guardian", "-N"]
9 changes: 0 additions & 9 deletions gitlabci/docker-ci/run.sh

This file was deleted.

54 changes: 54 additions & 0 deletions gitlabci/docker-hub-arm.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
docker-hub-build-arm:
stage: Docker-hub-build
image: docker:dind
dependencies:
- package:debarm
variables:
DOCKER_DST: "/tmp/e2guardian/scripts/debian_package/e2debian_package"
tags:
- arm
before_script:
- docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_TOKEN" $DOCKER_HUB_REGISTRY
script:
- cd $CI_PROJECT_DIR && cp gitlabci/docker-ci/Dockerfile ./Dockerfile
- docker build --build-arg E2_VERSION="$CI_COMMIT_BRANCH" --pull -t $CONTAINER_BUILD_NOPROD_NAME_ARM .
- docker push $CONTAINER_BUILD_NOPROD_NAME_ARM

variables:
CONTAINER_PROD_IMAGE_ARM: $HUB_REGISTRY_IMAGE:$CI_COMMIT_BRANCH-arm
CONTAINER_BUILD_NOPROD_NAME_ARM: $HUB_REGISTRY_IMAGE:build-noprod-arm

docker-hub-test-arm:
image: $CONTAINER_BUILD_NOPROD_NAME_ARM
stage: Docker-hub-test
dependencies: []
script:
- e2guardian -N &
- sleep 5 && export https_proxy=http://localhost:8080 && curl -k https://www.google.fr
tags:
- arm

dive-arm:
image:
name: wagoodman/dive:latest
entrypoint: [""]
dependencies: []
stage: Docker-hub-test
script:
- docker pull $CONTAINER_BUILD_NOPROD_NAME_ARM
- dive $CONTAINER_BUILD_NOPROD_NAME_ARM
variables:
CI: "true"

push-docker-hub-arm:
stage: Docker-hub-pushtag
image: docker:dind
dependencies: []
before_script:
- docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_TOKEN" $DOCKER_HUB_REGISTRY
script:
- docker pull $CONTAINER_BUILD_NOPROD_NAME_ARM
- docker tag $CONTAINER_BUILD_NOPROD_NAME_ARM $CONTAINER_PROD_IMAGE_ARM
- docker push $CONTAINER_PROD_IMAGE_ARM
tags:
- arm
Loading

0 comments on commit d8da3f3

Please sign in to comment.