Skip to content

Commit a7e1939

Browse files
EricJoy2048EricJoy2048
authored
Fix file change not be check when CI run in fork repository (apache#5515)
1 parent 980d38e commit a7e1939

File tree

3 files changed

+154
-41
lines changed

3 files changed

+154
-41
lines changed

‎.github/workflows/backend.yml

+53-40
Original file line numberDiff line numberDiff line change
@@ -80,48 +80,61 @@ jobs:
8080
ut-modules: ${{ steps.ut-modules.outputs.modules }}
8181
it-modules: ${{ steps.it-modules.outputs.modules }}
8282
steps:
83-
- uses: actions/checkout@v3 # required for push event
83+
- uses: actions/checkout@v4
84+
with:
85+
fetch-depth: '2000'
86+
- name: checkout apache seatunnel dev branch
87+
id: git_init
88+
run: |
89+
/usr/bin/git remote add apache https://github.com/apache/seatunnel
90+
/usr/bin/git -c protocol.version=2 fetch --no-tags --prune --no-recurse-submodules --depth=2000 apache +refs/heads/dev*:refs/remotes/apache/dev* +refs/tags/dev*:refs/tags/dev*
91+
/usr/bin/git checkout apache/dev
92+
/usr/bin/git checkout '${{ github.ref }}'
93+
echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
8494
- uses: actions/setup-python@v4
8595
with:
8696
python-version: '3.11.0'
87-
- name: Check for file changes
88-
uses: dorny/paths-filter@v2
97+
- name: Check for file changes by python
8998
id: filter
90-
with:
91-
token: ${{ github.token }}
92-
list-files: json
93-
# The following is a single composite pattern that allows next CI steps,
94-
# the pattern is in form of [not (foo or bar)] to be safe.
95-
# When new files come in, the CI will NOT ignore them unless listed,
96-
# so remember to extend here if they do not serve functional purposes.
97-
# NOTE: careful with using ** in expression, keep !**/{old, new things}.
98-
filters: |
99-
cv2:
100-
- "seatunnel-connectors-v2/**"
101-
cv2-e2e:
102-
- "seatunnel-e2e/seatunnel-connector-v2-e2e/**"
103-
api:
104-
- "seatunnel-api/**"
105-
- "seatunnel-common/**"
106-
- "seatunnel-config/**"
107-
- "seatunnel-connectors/**"
108-
- "seatunnel-core/**"
109-
- "seatunnel-e2e/seatunnel-e2e-common/**"
110-
- "seatunnel-formats/**"
111-
- "seatunnel-plugin-discovery/**"
112-
- "seatunnel-transforms-v2/**"
113-
- "seatunnel-translation/**"
114-
- "seatunnel-e2e/seatunnel-transforms-v2-e2e/**"
115-
- "seatunnel-connectors/**"
116-
- "pom.xml"
117-
- "**/workflows/**"
118-
- "**/tools/**"
119-
engine:
120-
- "seatunnel-engine/**"
121-
engine-e2e:
122-
- "seatunnel-e2e/seatunnel-engine-e2e/**"
123-
deleted-poms:
124-
- deleted: "**/pom.xml"
99+
run: |
100+
current_branch='${{ steps.git_init.outputs.branch }}'
101+
pip install GitPython
102+
workspace="${GITHUB_WORKSPACE}"
103+
cv2_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-connectors-v2/**"`
104+
true_or_false=${cv2_files%%$'\n'*}
105+
file_list=${cv2_files#*$'\n'}
106+
echo "cv2=$true_or_false" >> $GITHUB_OUTPUT
107+
echo "cv2_files=$file_list" >> $GITHUB_OUTPUT
108+
109+
cv2_e2e_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-e2e/seatunnel-connector-v2-e2e/**"`
110+
true_or_false=${cv2_e2e_files%%$'\n'*}
111+
file_list=${cv2_e2e_files#*$'\n'}
112+
echo "cv2-e2e=$true_or_false" >> $GITHUB_OUTPUT
113+
echo "cv2-e2e_files=$file_list" >> $GITHUB_OUTPUT
114+
115+
engine_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-engine/**"`
116+
true_or_false=${engine_files%%$'\n'*}
117+
file_list=${engine_files#*$'\n'}
118+
echo "engine=$true_or_false" >> $GITHUB_OUTPUT
119+
echo "engine_files=$file_list" >> $GITHUB_OUTPUT
120+
121+
deleted_poms_files=`python tools/update_modules_check/check_file_updates.py d $workspace apache/dev origin/$current_branch "**/pom.xml"`
122+
true_or_false=${deleted_poms_files%%$'\n'*}
123+
file_list=${deleted_poms_files#*$'\n'}
124+
echo "deleted-poms=$true_or_false" >> $GITHUB_OUTPUT
125+
echo "deleted-poms_files=$file_list" >> $GITHUB_OUTPUT
126+
127+
engine_e2e_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-e2e/seatunnel-engine-e2e/**"`
128+
true_or_false=${engine_e2e_files%%$'\n'*}
129+
file_list=${engine_e2e_files#*$'\n'}
130+
echo "engine-e2e=$true_or_false" >> $GITHUB_OUTPUT
131+
echo "engine-e2e_files=$file_list" >> $GITHUB_OUTPUT
132+
133+
api_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-api/**" "seatunnel-common/**" "seatunnel-config/**" "seatunnel-connectors/**" "seatunnel-core/**" "seatunnel-e2e/seatunnel-e2e-common/**" "seatunnel-formats/**" "seatunnel-plugin-discovery/**" "seatunnel-transforms-v2/**" "seatunnel-translation/**" "seatunnel-e2e/seatunnel-transforms-v2-e2e/**" "seatunnel-connectors/**" "pom.xml" "**/workflows/**" "tools/**"`
134+
true_or_false=${api_files%%$'\n'*}
135+
file_list=${api_files#*$'\n'}
136+
echo "api=$true_or_false" >> $GITHUB_OUTPUT
137+
echo "api_files=$file_list" >> $GITHUB_OUTPUT
125138
126139
- name: Check Connector V2 Update
127140
id: cv2-modules
@@ -257,7 +270,7 @@ jobs:
257270
matrix:
258271
java: [ '8', '11' ]
259272
os: [ 'ubuntu-latest', 'windows-latest' ]
260-
timeout-minutes: 36
273+
timeout-minutes: 60
261274
steps:
262275
- uses: actions/checkout@v2
263276
- name: Set up JDK ${{ matrix.java }}
@@ -902,4 +915,4 @@ jobs:
902915
run: |
903916
./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-rocketmq-e2e -am -Pci
904917
env:
905-
MAVEN_OPTS: -Xmx4096m
918+
MAVEN_OPTS: -Xmx4096m

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -49,4 +49,5 @@ spark-warehouse
4949
*.flattened-pom.xml
5050

5151
seatunnel-examples
52-
/lib/*
52+
/lib/*
53+
version.properties

‎tools/update_modules_check/check_file_updates.py

+99
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# !/usr/bin/python
17+
import argparse
18+
import git
19+
import json
20+
import glob
21+
22+
def get_changed_files_between_branches(repo_path1, branch1, branch2, directorys):
23+
repo1 = git.Repo(repo_path1)
24+
25+
commit1 = repo1.commit(branch1)
26+
commit2 = repo1.commit(branch2)
27+
28+
diff = commit1.diff(commit2, create_patch=True)
29+
30+
changed_files = []
31+
32+
for file_diff in diff:
33+
for directory in directorys:
34+
if file_diff.a_path != file_diff.b_path:
35+
if file_diff.b_path is not None and glob.fnmatch.fnmatch(file_diff.b_path, directory):
36+
changed_files.append(file_diff.b_path)
37+
38+
if file_diff.a_path is not None and glob.fnmatch.fnmatch(file_diff.a_path, directory):
39+
changed_files.append(file_diff.a_path)
40+
else:
41+
if glob.fnmatch.fnmatch(file_diff.b_path, directory):
42+
changed_files.append(file_diff.b_path)
43+
44+
return changed_files
45+
46+
def get_deleted_files_between_branches(repo_path, branch1, branch2, directorys):
47+
deleted_files = []
48+
49+
repo1 = git.Repo(repo_path)
50+
51+
commit1 = repo1.commit(branch1)
52+
commit2 = repo1.commit(branch2)
53+
54+
diff = commit1.diff(commit2, create_patch=True)
55+
56+
for file_diff in diff:
57+
for directory in directorys:
58+
if file_diff.a_path is not None and file_diff.b_path is None and glob.fnmatch.fnmatch(file_diff.a_path, directory):
59+
deleted_files.append(file_diff.a_path)
60+
61+
return deleted_files
62+
63+
if __name__ == "__main__":
64+
parser = argparse.ArgumentParser(description="Compare changes in a specified directory between two branches in different repositories.")
65+
parser.add_argument("type", help="ua will return update and add files, d will return delete files")
66+
parser.add_argument("repo_path", help="Path to the first local Git repository")
67+
parser.add_argument("branch1", help="Name of the first branch to compare")
68+
parser.add_argument("branch2", help="Name of the second branch to compare")
69+
parser.add_argument("directorys", nargs="+", help="Directory to compare")
70+
71+
args = parser.parse_args()
72+
73+
repo = git.Repo(args.repo_path)
74+
75+
ref1 = repo.refs[args.branch1]
76+
ref2 = repo.refs[args.branch2]
77+
78+
common_ancestor = repo.merge_base(ref1, ref2)[0].hexsha
79+
80+
if args.type == 'ua':
81+
changed_files = get_changed_files_between_branches(args.repo_path, common_ancestor, args.branch2, args.directorys)
82+
if changed_files:
83+
print('true')
84+
result = json.dumps(changed_files, indent=None)
85+
print(result)
86+
else:
87+
print('false')
88+
result = json.dumps([], indent=None)
89+
print(result)
90+
else:
91+
delete_files = get_deleted_files_between_branches(args.repo_path, common_ancestor, args.branch2, args.directorys)
92+
if delete_files:
93+
print('true')
94+
result = json.dumps(delete_files, indent=None)
95+
print(result)
96+
else:
97+
print('false')
98+
result = json.dumps([], indent=None)
99+
print(result)

0 commit comments

Comments
 (0)