Copy files from OCI Buckets and sent do AWS S3 with GNU Parallel based on ids.txt info line by line. Need OCI CLI and AWS CLI installed and configured.

copy.sh

#!/bin/bash 

# Define the input file name
INPUT_FILE="ids.txt"

# Check if the file exists
if [[ ! -f "$INPUT_FILE" ]]; then
    echo "Error: File '$INPUT_FILE' not found."
    exit 1
fi

copy() {
  item_item="$1"
  cmdOci="oci os object bulk-download -ns XXXXXXX -bn mybucket --prefix="datapart/$item_item" --download-dir ./files/"
  DIR="./files/datapart/$item_item/"
  cmdMkDir="mkdir $DIR"
  cmdAws="aws s3 cp $DIR s3://newbucket/datapart/$item_item/ --recursive"
  cmdRm="rm -rf $DIR"

  echo "Processing item: $item_item"
  echo "$DIR"
  eval "$cmdMkDir"
  eval "$cmdOci"
  eval "$cmdAws"
  eval "$cmdRm"

  if [ -d "$DIR" ]; then
      echo "Folder exists: $DIR"
  else
      echo "Finished $item_item"
  fi
}

export -f copy

cat $INPUT_FILE | parallel -j0 copy

ids.txt

1
2
3
4
5
100