Skip to content

Commit

Permalink
Add Python39 runtime (kyma-project#11498)
Browse files Browse the repository at this point in the history
* Add Python39 runtime

* doc update

* update alpine for python39

* Update docs/serverless/03-01-function-specification.md

Co-authored-by: Karolina Zydek <[email protected]>

* Update docs/serverless/03-04-git-source-type.md

Co-authored-by: Karolina Zydek <[email protected]>

* cleanup

* fix typo

* fix docs typo

Co-authored-by: Karolina Zydek <[email protected]>
  • Loading branch information
moelsayed and Karolina Zydek authored Jul 5, 2021
1 parent 7f9165c commit 28c618e
Show file tree
Hide file tree
Showing 22 changed files with 518 additions and 15 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ spec:
- nodejs12
- nodejs14
- python38
- python39
type: string
source:
description: Source defines the source code of a function
Expand Down Expand Up @@ -282,6 +283,7 @@ spec:
- nodejs14
- python38
- nodejs10
- python39
type: string
source:
type: string
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
apiVersion: serverless.kyma-project.io/v1alpha1
kind: Function
metadata:
name: python39-sample-fn
spec:
runtime: python39
source: |
def main(event, context):
return "Hello world from Python"
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,7 @@ func (r *FunctionReconciler) getPackageConfigVolumeMountsForRuntime(rtm serverle
switch rtm {
case serverlessv1alpha1.Nodejs12, serverlessv1alpha1.Nodejs14:
return []corev1.VolumeMount{{Name: "registry-config", ReadOnly: true, MountPath: path.Join(workspaceMountPath, "registry-config/.npmrc"), SubPath: ".npmrc"}}
case serverlessv1alpha1.Python38:
case serverlessv1alpha1.Python38, serverlessv1alpha1.Python39:
return []corev1.VolumeMount{{Name: "registry-config", ReadOnly: true, MountPath: path.Join(workspaceMountPath, "registry-config/pip.conf"), SubPath: "pip.conf"}}
}
return nil
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -484,6 +484,25 @@ func TestFunctionReconciler_buildJob(t *testing.T) {
{Name: "registry-config", MountPath: "/workspace/registry-config/pip.conf", SubPath: "pip.conf", ReadOnly: true},
},
},
{
Name: "Success Python39",
Runtime: serverlessv1alpha1.Python39,
ExpectedVolumesLen: 4,
ExpectedVolumes: []expectedVolume{
{name: "sources", localObjectReference: cmName},
{name: "runtime", localObjectReference: rtmCfg.DockerfileConfigMapName},
{name: "credentials", localObjectReference: dockerCfg.ActiveRegistryConfigSecretName},
{name: "registry-config", localObjectReference: r.config.PackageRegistryConfigSecretName},
},
ExpectedMountsLen: 5,
ExpectedVolumeMounts: []corev1.VolumeMount{
{Name: "sources", MountPath: "/workspace/src/deps.txt", SubPath: FunctionDepsKey, ReadOnly: true},
{Name: "sources", MountPath: "/workspace/src/function.abap", SubPath: FunctionSourceKey, ReadOnly: true},
{Name: "runtime", MountPath: "/workspace/Dockerfile", SubPath: "Dockerfile", ReadOnly: true},
{Name: "credentials", MountPath: "/docker", ReadOnly: true},
{Name: "registry-config", MountPath: "/workspace/registry-config/pip.conf", SubPath: "pip.conf", ReadOnly: true},
},
},
}

for _, testCase := range testCases {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,15 @@ func GetRuntimeConfig(r serverlessv1alpha1.Runtime) Config {
RuntimeEnvs: []corev1.EnvVar{{Name: "PYTHONPATH", Value: "$(KUBELESS_INSTALL_VOLUME)/lib.python3.8/site-packages:$(KUBELESS_INSTALL_VOLUME)"}, // https://github.com/kubeless/runtimes/blob/master/stable/python/python.jsonnet#L45
{Name: "FUNC_RUNTIME", Value: "python38"}},
}
case serverlessv1alpha1.Python39:
return Config{
Runtime: serverlessv1alpha1.Python39,
DependencyFile: "requirements.txt",
FunctionFile: "handler.py",
DockerfileConfigMapName: "dockerfile-python-39",
RuntimeEnvs: []corev1.EnvVar{{Name: "PYTHONPATH", Value: "$(KUBELESS_INSTALL_VOLUME)/lib.python3.9/site-packages:$(KUBELESS_INSTALL_VOLUME)"}, // https://github.com/kubeless/runtimes/blob/master/stable/python/python.jsonnet#L45
{Name: "FUNC_RUNTIME", Value: "python39"}},
}
default:
return Config{
Runtime: serverlessv1alpha1.Nodejs12,
Expand All @@ -65,7 +74,7 @@ func GetRuntime(r serverlessv1alpha1.Runtime) Runtime {
switch r {
case serverlessv1alpha1.Nodejs12, serverlessv1alpha1.Nodejs14:
return nodejs{}
case serverlessv1alpha1.Python38:
case serverlessv1alpha1.Python38, serverlessv1alpha1.Python39:
return python{}
default:
return nodejs{}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,18 @@ func TestGetRuntimeConfig(t *testing.T) {
{Name: "FUNC_RUNTIME", Value: "python38"}},
},
},
"python39": {
name: "python39",
runtime: serverlessv1alpha1.Python39,
want: runtime.Config{
Runtime: serverlessv1alpha1.Python39,
DependencyFile: "requirements.txt",
FunctionFile: "handler.py",
DockerfileConfigMapName: "dockerfile-python-39",
RuntimeEnvs: []corev1.EnvVar{{Name: "PYTHONPATH", Value: "$(KUBELESS_INSTALL_VOLUME)/lib.python3.9/site-packages:$(KUBELESS_INSTALL_VOLUME)"},
{Name: "FUNC_RUNTIME", Value: "python39"}},
},
},
"nodej14": {
name: "nodejs14 config",
runtime: serverlessv1alpha1.Nodejs14,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,27 @@ const (

// Runtime enumerates runtimes that are currently supported by Function Controller
// It is a subset of RuntimeExtended
// +kubebuilder:validation:Enum=nodejs12;nodejs14;python38
// +kubebuilder:validation:Enum=nodejs12;nodejs14;python38;python39
type Runtime string

const (
Nodejs12 Runtime = "nodejs12"
Nodejs14 Runtime = "nodejs14"
Python38 Runtime = "python38"
Python39 Runtime = "python39"
)

// RuntimeExtended enumerates runtimes that are either currently supported or
// no longer supported but there still might be "read-only" Functions using them
// +kubebuilder:validation:Enum=nodejs12;nodejs14;python38;nodejs10
// +kubebuilder:validation:Enum=nodejs12;nodejs14;python38;nodejs10;python39
type RuntimeExtended string

const (
RuntimeExtendedNodejs10 RuntimeExtended = "nodejs10"
RuntimeExtendedNodejs12 RuntimeExtended = "nodejs12"
RuntimeExtendedNodejs14 RuntimeExtended = "nodejs14"
RuntimeExtendedPython38 RuntimeExtended = "python38"
RuntimeExtendedPython39 RuntimeExtended = "python39"
)

const (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ func ValidateDependencies(runtime Runtime, dependencies string) error {
switch runtime {
case Nodejs12, Nodejs14:
return validateNodeJSDependencies(dependencies)
case Python38:
case Python38, Python39:
return nil
}

Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 17 additions & 3 deletions components/function-runtimes/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ release: build-images push-images
.PHONY: build-images
build-images: build-function-runtime-nodejs12-image \
build-function-runtime-nodejs14-image \
build-function-runtime-python38-image
build-function-runtime-python38-image \
build-function-runtime-python39-image

.PHONY: push-images
push-images: push-function-runtime-nodejs12-image \
push-function-runtime-nodejs14-image \
push-function-runtime-python38-image
push-function-runtime-python38-image \
push-function-runtime-python39-image

## nodejs12
FUNCTION_RUNTIME_IMAGE_NODEJS12 = function-runtime-nodejs12
Expand Down Expand Up @@ -46,4 +48,16 @@ build-function-runtime-python38-image:

push-function-runtime-python38-image:
docker tag $(FUNCTION_RUNTIME_IMAGE_PYTHON38) $(FUNCTION_RUNTIME_PYTHON38_IMG_NAME):$(DOCKER_TAG)
docker push $(FUNCTION_RUNTIME_PYTHON38_IMG_NAME):$(DOCKER_TAG)
docker push $(FUNCTION_RUNTIME_PYTHON38_IMG_NAME):$(DOCKER_TAG)

## python39
FUNCTION_RUNTIME_IMAGE_PYTHON39 = function-runtime-python39
FUNCTION_RUNTIME_PYTHON39_IMG_NAME:=$(DOCKER_PUSH_REPOSITORY)$(DOCKER_PUSH_DIRECTORY)/$(FUNCTION_RUNTIME_IMAGE_PYTHON39)

.PHONY: build-function-runtime-python39-image push-function-runtime-python39-image
build-function-runtime-python39-image:
docker build -t $(FUNCTION_RUNTIME_IMAGE_PYTHON39) $(ROOT)/python39/

push-function-runtime-python39-image:
docker tag $(FUNCTION_RUNTIME_IMAGE_PYTHON39) $(FUNCTION_RUNTIME_PYTHON39_IMG_NAME):$(DOCKER_TAG)
docker push $(FUNCTION_RUNTIME_PYTHON39_IMG_NAME):$(DOCKER_TAG)
161 changes: 161 additions & 0 deletions components/function-runtimes/python39/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
#
# NOTE: THIS DOCKERFILE IS GENERATED VIA "update.sh"
#
# PLEASE DO NOT EDIT IT DIRECTLY.
#

# We use this code to manually use newest alpine and nodejs.
# Use official build when it will be available.

FROM alpine:3.13.5

# ensure local python is preferred over distribution python
ENV PATH /usr/local/bin:$PATH

# http://bugs.python.org/issue19846
# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
ENV LANG C.UTF-8

# runtime dependencies
RUN set -eux; \
apk add --no-cache \
# install ca-certificates so that HTTPS works consistently
ca-certificates \
;
# other runtime dependencies for Python are installed later

ENV GPG_KEY E3FF2839C048B25C084DEBE9B26995E310250568
ENV PYTHON_VERSION 3.9.5


# expat-dev for alpine:3.13.5 is outdated and vulnerable. We will install it from edge .
# We can't upgrade fully to 3.14 because it has a dependancy on a specific recent docker version https://gitlab.alpinelinux.org/alpine/aports/-/issues/12396

RUN set -ex \
&& apk add --no-cache --virtual .fetch-deps --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main \
gnupg \
tar \
xz \
expat-dev \
\
&& wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
&& wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
&& export GNUPGHOME="$(mktemp -d)" \
&& gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY" \
&& gpg --batch --verify python.tar.xz.asc python.tar.xz \
&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \
&& rm -rf "$GNUPGHOME" python.tar.xz.asc \
&& mkdir -p /usr/src/python \
&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \
&& rm python.tar.xz \
\
&& apk add --no-cache --virtual .build-deps \
bluez-dev \
bzip2-dev \
coreutils \
dpkg-dev dpkg \
findutils \
gcc \
gdbm-dev \
libc-dev \
libffi-dev \
libnsl-dev \
libtirpc-dev \
linux-headers \
make \
ncurses-dev \
openssl-dev \
pax-utils \
readline-dev \
sqlite-dev \
tcl-dev \
tk \
tk-dev \
util-linux-dev \
xz-dev \
zlib-dev \
# add build deps before removing fetch deps in case there's overlap
&& apk del --no-network .fetch-deps \
\
&& cd /usr/src/python \
&& gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
&& ./configure \
--build="$gnuArch" \
--enable-loadable-sqlite-extensions \
--enable-optimizations \
--enable-option-checking=fatal \
--enable-shared \
--with-system-expat \
--with-system-ffi \
--without-ensurepip \
&& make -j "$(nproc)" \
# set thread stack size to 1MB so we don't segfault before we hit sys.getrecursionlimit()
# https://github.com/alpinelinux/aports/commit/2026e1259422d4e0cf92391ca2d3844356c649d0
EXTRA_CFLAGS="-DTHREAD_STACK_SIZE=0x100000" \
LDFLAGS="-Wl,--strip-all" \
&& make install \
&& rm -rf /usr/src/python \
\
&& find /usr/local -depth \
\( \
\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
-o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) \) \
-o \( -type f -a -name 'wininst-*.exe' \) \
\) -exec rm -rf '{}' + \
\
&& find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec scanelf --needed --nobanner --format '%n#p' '{}' ';' \
| tr ',' '\n' \
| sort -u \
| awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
| xargs -rt apk add --no-cache --virtual .python-rundeps \
&& apk del --no-network .build-deps \
\
&& python3 --version

# make some useful symlinks that are expected to exist
RUN cd /usr/local/bin \
&& ln -s idle3 idle \
&& ln -s pydoc3 pydoc \
&& ln -s python3 python \
&& ln -s python3-config python-config

# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
ENV PYTHON_PIP_VERSION 21.0.1
# https://github.com/pypa/get-pip
ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/29f37dbe6b3842ccd52d61816a3044173962ebeb/public/get-pip.py
ENV PYTHON_GET_PIP_SHA256 e03eb8a33d3b441ff484c56a436ff10680479d4bd14e59268e67977ed40904de

RUN set -ex; \
\
wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \
echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c -; \
\
python get-pip.py \
--disable-pip-version-check \
--no-cache-dir \
"pip==$PYTHON_PIP_VERSION" \
; \
pip --version; \
\
find /usr/local -depth \
\( \
\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
-o \
\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
\) -exec rm -rf '{}' +; \
rm -f get-pip.py

# Serverless

LABEL source = [email protected]:kyma-project/kyma.git

COPY kubeless/requirements.txt /kubeless/requirements.txt
RUN pip install -r /kubeless/requirements.txt

COPY kubeless/ /

WORKDIR /

USER 1000

CMD ["python", "/kubeless.py"]
2 changes: 2 additions & 0 deletions components/function-runtimes/python39/kubeless/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
venv
handler.py
Loading

0 comments on commit 28c618e

Please sign in to comment.