Add a new compute model to Feathr (feathr-ai#820)

* Add working gradle build * Set up pdl support * Working PDL java code gen * With pdl files from metadata models * With pdl files from compute model * Fix compile for all pdl files * Add working gradle build * Migrate frame-config module into feathr * Migrate fcm graph module to feathr * Add FCM offline execution code, includes FDS metadata code * Add needed jars for feathr-config tests * Switch client to FeathrClient2 for local tests and fix config errors * Fix SWA test * Add gradle wrapper jar * Change name of git PR test from sbt to gradle * Switch python client to use FCM client * Exclude json from dependency * Add hacky solution to handle json dependency conflict in cloud * Add json to local dependency * Add log to debug cloud jar * Add json as dependency * Another attempt at resolving json dependency * Resolve json via shading * Fix json shading * Remove log * Shade typesafe config for cloud jar * Add maven publish code to build.gradle * Add working local maven build and rename frame-config to feathr-config to avoid namespace conflict * Modify sonatype creds * Change so no need to sign if releasing snapshot version * Update build.gradle to allow publishing of all modules * Removed FDS handling from Feathr * All tests working * Deleted FR stuff * Remove dimension and other tensor related stuff * Remove mlfeatureversionurn from defaultvalueresolver * Remove mlfeatureversionurn and featureref * Remove featuredefinition files * Remove featureRef and typedRef * final cleanup * Fix merge conflict bugs * Fix guava error * udf plugin for swa features * row-transformations optimization * fix bug * fix another bug * always execute agg nodes first * Add SWA log * reverse order of execution * group by datasource * Fix bug * Merge main into fcm branch * Remove insecure URLs * Add back removed files * Add back removed files * Add back removed files * Change PR build system to gradle * Change sbt job to gradle jobb * Change sbt workflow:wq * Update maven github workflow to use gradle * fix failing test * remove sbt project module * Remove sbt related files * Change docs to reflect gradle * Remove keywords * Create a single jar * 1. Fix jar not getting populated\n 2. Fix documentation bugs * pubishToMavenLocal Working * With FFE integrated * maven upload working * Update docs and code clean up * add gradle-wrapper file * Push all dependency jars * Update docs * Docs cleanup * Update github workflow commands * Update github workflow * Update workflow syntax * Update version * Add gradle version to github workflow * Update gradle version w/o quotes * Remove github gradle version * Github workflow fix * Github workflow fix-2 * Github workflow fix-4 Co-authored-by: Bozhong Hu <[email protected]> Co-authored-by: rkashyap <[email protected]>
JaeDukSeo · Nov 30, 2022 · 758f249 · 758f249
1 parent 654d56e
commit 758f249
Show file tree

Hide file tree

Showing 1,059 changed files with 31,643 additions and 619 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,6 @@
+#
+# https://help.github.com/articles/dealing-with-line-endings/
+#
+# These are explicitly windows files and should use crlf
+*.bat           text eol=crlf
+
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
@@ -1,5 +1,5 @@
 # This workflow builds the docker container and publishes to dockerhub with appropriate tag
-# It has two triggers, 
+# It has two triggers,
 #   1. daily i.e. runs everyday at specific time.
 #   2. Anytime a new branch is created under releases
 
@@ -22,19 +22,19 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
-      
+
       - name: Log in to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@v4
         with:
           images: feathrfeaturestore/feathr-registry
-      
+
       - name: Build and push Docker image
         uses: docker/build-push-action@v3
         with:
@@ -72,4 +72,4 @@ jobs:
         id: deploy-to-feathr-registry-sql-rbac
         uses: distributhor/[email protected]
         env:
-          webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_REGISTRY_SQL_RBAC_WEBHOOK }}
+          webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_REGISTRY_SQL_RBAC_WEBHOOK }}
diff --git a/.github/workflows/publish-to-maven.yml b/.github/workflows/publish-to-maven.yml
@@ -1,18 +1,18 @@
 name: Publish package to the Maven Central Repository
-on: 
+on:
   push:
     # This pipeline will get triggered everytime there is a new tag created.
-    # It is required 
+    # It is required
     tags: ["*"]
 
 jobs:
   publish-to-maven:
     runs-on: ubuntu-latest
-    
+
     steps:
       - name: Checkout source
         uses: actions/checkout@v2
-      
+
     # Setting up JDK 8, this is required to build Feathr
       - name: Set up JDK 8
         uses: actions/setup-java@v2
@@ -27,10 +27,9 @@ jobs:
 
     # CI release command defaults to publishSigned
     # Sonatype release command defaults to sonaTypeBundleRelease
-    # https://github.com/sbt/sbt-ci-release
-      - name: Sbt ci release
-        run: |
-          sbt ci-release
+      - name: Gradle publish
+        if: startsWith(github.head_ref, 'release/v')
+        run: gradle clean publish
         env:
           PGP_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
           PGP_SECRET: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}

diff --git a/.github/workflows/pull_request_push_test.yml b/.github/workflows/pull_request_push_test.yml
@@ -28,7 +28,7 @@ on:
     - cron: '00 13 * * *'
 
 jobs:
-  sbt_test:
+  gradle_test:
     runs-on: ubuntu-latest
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test'))
     steps:
@@ -41,7 +41,7 @@ jobs:
           java-version: "8"
           distribution: "temurin"
       - name: Run tests
-        run: sbt clean && sbt test
+        run: ./gradlew clean && ./gradlew test
 
   python_lint:
     runs-on: ubuntu-latest
@@ -75,15 +75,15 @@ jobs:
         with:
           java-version: "8"
           distribution: "temurin"
-      - name: Build JAR
+      - name: Gradle build
         run: |
-          sbt assembly
+          ./gradlew build
           # remote folder for CI upload
           echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_github_action_$(date +"%H_%M_%S")" >> $GITHUB_ENV
           # get local jar name without paths so version change won't affect it
-          echo "FEATHR_LOCAL_JAR_NAME=$(ls target/scala-2.12/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_NAME=$(ls build/libs/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
           # get local jar name without path
-          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls target/scala-2.12/*.jar)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls build/libs/*.jar)" >> $GITHUB_ENV
       - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:
@@ -142,15 +142,16 @@ jobs:
         with:
           java-version: "8"
           distribution: "temurin"
-      - name: Build JAR
+
+      - name: Gradle build
         run: |
-          sbt assembly
+          ./gradlew build
           # remote folder for CI upload
           echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_github_action_$(date +"%H_%M_%S")" >> $GITHUB_ENV
           # get local jar name without paths so version change won't affect it
-          echo "FEATHR_LOCAL_JAR_NAME=$(ls target/scala-2.12/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_NAME=$(ls build/libs/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
           # get local jar name without path
-          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls target/scala-2.12/*.jar)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls build/libs/*.jar)" >> $GITHUB_ENV
       - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:
@@ -210,15 +211,16 @@ jobs:
         with:
           java-version: "8"
           distribution: "temurin"
-      - name: Build JAR
+
+      - name: Gradle build
         run: |
-          sbt assembly
+          ./gradlew build
           # remote folder for CI upload
           echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_github_action_$(date +"%H_%M_%S")" >> $GITHUB_ENV
           # get local jar name without paths so version change won't affect it
-          echo "FEATHR_LOCAL_JAR_NAME=$(ls target/scala-2.12/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_NAME=$(ls build/libs/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
           # get local jar name without path
-          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls target/scala-2.12/*.jar)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls build/libs/*.jar)" >> $GITHUB_ENV
       - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:
@@ -258,7 +260,7 @@ jobs:
 
   failure_notification:
     # If any failure, warning message will be sent
-    needs: [sbt_test, python_lint, databricks_test, azure_synapse_test, local_spark_test]
+    needs: [gradle_test, python_lint, databricks_test, azure_synapse_test, local_spark_test]
     runs-on: ubuntu-latest
     if: failure() && github.event_name == 'schedule'
     steps:
@@ -268,12 +270,12 @@ jobs:
 
   notification:
     # Final Daily Report with all job status
-    needs: [sbt_test, python_lint, databricks_test, azure_synapse_test, local_spark_test]
+    needs: [gradle_test, python_lint, databricks_test, azure_synapse_test, local_spark_test]
     runs-on: ubuntu-latest
     if: always() && github.event_name == 'schedule'
     steps:
       - name: Get Date
         run: echo "NOW=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
       - name: Notification
         run: |
-          curl -H 'Content-Type: application/json' -d '{"text": "${{env.NOW}} Daily Report: 1. SBT Test ${{needs.sbt_test.result}}, 2. Python Lint Test ${{needs.python_lint.result}}, 3. Databricks Test ${{needs.databricks_test.result}}, 4. Synapse Test ${{needs.azure_synapse_test.result}} , 5. LOCAL SPARK TEST ${{needs.local_spark_test.result}}. Link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' ${{ secrets.TEAMS_WEBHOOK }}
+          curl -H 'Content-Type: application/json' -d '{"text": "${{env.NOW}} Daily Report: 1. Gradle Test ${{needs.gradle_test.result}}, 2. Python Lint Test ${{needs.python_lint.result}}, 3. Databricks Test ${{needs.databricks_test.result}}, 4. Synapse Test ${{needs.azure_synapse_test.result}} , 5. LOCAL SPARK TEST ${{needs.local_spark_test.result}}. Link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' ${{ secrets.TEAMS_WEBHOOK }}
diff --git a/.gitignore b/.gitignore
@@ -3,7 +3,7 @@
 .AppleDouble
 .LSOverride
 metastore_db
-src/integTest
+feathr-impl/src/integTest
 test-output
 temp
 
@@ -189,30 +189,37 @@ cython_debug/
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*
 
-target/
 .idea
 .project/target
 .project/project
 .DS_store
-.DS_Store
 *.jar
-src/main/scala/META-INF/MANIFEST.MF
+feathr-impl/src/main/scala/META-INF/MANIFEST.MF
 *.MF
 feathr_project/feathr_cli.egg-info/*
 *.pyc
+*.iml
 
 # VS Code
 .vscode
 
 #Local Build
 null/*
 
+# Ignore Gradle project-specific cache directory
+.gradle
+
+# Ignore Gradle build output directory
+build
+
 # For Metal Server
 .metals/
 .bloop/
 project/.bloop
 metals.sbt
+
 .bsp/sbt.json
 
 # Feathr output debug folder
 **/debug/
+
diff --git a/.husky/pre-commit b/.husky/pre-commit
@@ -1,4 +1,4 @@
 #!/usr/bin/env sh
 . "$(dirname -- "$0")/_/husky.sh"
 
-npx lint-staged
+npx lint-staged