Skip to content

Commit

Permalink
Permission & copy change to run on serverless
Browse files Browse the repository at this point in the history
CMD 6: Change to permission on /tmp/data/*
CMD 8, 10: Change to the dbutils.fs.cp to %sh based cp
  • Loading branch information
lingak2311 committed Dec 20, 2024
1 parent 447be09 commit b38c53d
Showing 1 changed file with 28 additions and 3 deletions.
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
# Databricks notebook source
# DBTITLE 1,fetch the volume_folder value
dbutils.widgets.text("volume_folder", "")
volume_folder = dbutils.widgets.get("volume_folder")

# COMMAND ----------

# DBTITLE 1,install awscli
# MAGIC %pip install awscli

# COMMAND ----------

# DBTITLE 1,set the tmp dir and download the data from aws s3
# MAGIC %sh
# MAGIC mkdir -p /tmp/data
# MAGIC aws s3 cp --no-progress --no-sign-request s3://amazon-visual-anomaly/VisA_20220922.tar /tmp

# COMMAND ----------

# DBTITLE 1,unzip the images
# MAGIC %sh
# MAGIC mkdir -p /tmp/data
# MAGIC tar xf /tmp/VisA_20220922.tar --no-same-owner -C /tmp/data/
Expand All @@ -25,9 +29,30 @@

# COMMAND ----------

# DBTITLE 1,chmod permissions
# MAGIC %sh
# MAGIC chmod -R 777 /tmp/data/pcb1/Data/Images/

# COMMAND ----------

# DBTITLE 1,recreate images dir required for demo
dbutils.fs.rm(volume_folder + "/images", recurse=True)
dbutils.fs.rm(volume_folder + "/labels", recurse=True)
dbutils.fs.mkdirs(volume_folder + "/images")
dbutils.fs.cp("file:/tmp/data/pcb1/Data/Images/", volume_folder + "/images", recurse=True)

# COMMAND ----------

# DBTITLE 1,copy images data to volume
# MAGIC %sh
# MAGIC cp -r /tmp/data/pcb1/Data/Images/* /Volumes/main__build/dbdemos_computer_vision_dl/pcb_training_data/images/

# COMMAND ----------

# DBTITLE 1,recreate labels dir required for demo
dbutils.fs.rm(volume_folder + "/labels", recurse=True)
dbutils.fs.mkdirs(volume_folder + "/labels")
dbutils.fs.cp("file:/tmp/data/pcb1/image_anno.csv/", volume_folder + "/labels")

# COMMAND ----------

# DBTITLE 1,copy annotation data to volume
# MAGIC %sh
# MAGIC cp -r /tmp/data/pcb1/image_anno.csv /Volumes/main__build/dbdemos_computer_vision_dl/pcb_training_data/labels/

0 comments on commit b38c53d

Please sign in to comment.