Develop Oracle Cloud Infrastructure Data Flow Applications Locally, Deploy to The Cloud

import com.oracle.bmc.auth.ConfigFileAuthenticationDetailsProvider;
import com.oracle.bmc.ConfigFileReader;

//If your key is encrypted call setPassPhrase:
ConfigFileAuthenticationDetailsProvider authenticationDetailsProvider = new ConfigFileAuthenticationDetailsProvider(ConfigFileReader.DEFAULT_FILE_PATH, "<DEFAULT>"); 
configuration.put("fs.oci.client.auth.tenantId", authenticationDetailsProvider.getTenantId());
configuration.put("fs.oci.client.auth.userId", authenticationDetailsProvider.getUserId());
configuration.put("fs.oci.client.auth.fingerprint", authenticationDetailsProvider.getFingerprint());

String guessedPath = new File(configurationFilePath).getParent() + File.separator + "oci_api_key.pem";
configuration.put("fs.oci.client.auth.pemfilepath", guessedPath);

// Set the storage endpoint:
String region = authenticationDetailsProvider.getRegion().getRegionId();
String hostName = MessageFormat.format("https://objectstorage.{0}.oraclecloud.com", new Object[] { region });
configuration.put("fs.oci.client.hostname", hostName);

import os 
from pyspark  import SparkConf, SparkContext
from pyspark.sql  import SparkSession
# Check to see if we're in Data Flow or not.
if os.environ.get("HOME") == "/home/dataflow":
    spark_session = SparkSession.builder.appName("app").getOrCreate()
else:
    conf = SparkConf()
    oci_config = oci.config.from_file(oci.config.DEFAULT_LOCATION, "<DEFAULT>")
    conf.set("fs.oci.client.auth.tenantId", oci_config["tenancy"])
    conf.set("fs.oci.client.auth.userId", oci_config["user"])
    conf.set("fs.oci.client.auth.fingerprint", oci_config["fingerprint"])
    conf.set("fs.oci.client.auth.pemfilepath", oci_config["key_file"])
    conf.set(
        "fs.oci.client.hostname",
        "https://objectstorage.{0}.oraclecloud.com".format(oci_config["region"]),
    )

    spark_builder = SparkSession.builder.appName("app")
    spark_builder.config(conf=conf)
    spark_session = spark_builder.getOrCreate()

spark_context = spark_session.sparkContext

#!/bin/sh

CONFIG=$HOME/.oci/config
USER=$(egrep ' user' $CONFIG | cut -f2 -d=)
FINGERPRINT=$(egrep ' fingerprint' $CONFIG | cut -f2 -d=)
KEYFILE=$(egrep ' key_file' $CONFIG | cut -f2 -d=)
TENANCY=$(egrep ' tenancy' $CONFIG | cut -f2 -d=)
REGION=$(egrep ' region' $CONFIG | cut -f2 -d=)
REMOTEHOST="https://objectstorage.$REGION.oraclecloud.com"

spark-sql \
	--hiveconf fs.oci.client.auth.tenantId=$TENANCY \
	--hiveconf fs.oci.client.auth.userId=$USER \
	--hiveconf fs.oci.client.auth.fingerprint=$FINGERPRINT \
	--hiveconf fs.oci.client.auth.pemfilepath=$KEYFILE \
	--hiveconf fs.oci.client.hostname=$REMOTEHOST \
	-f script.sql

<properties>
            <oci-java-sdk-version>1.25.2</oci-java-sdk-version>
            </properties>
            
            <dependencies>
            <dependency>
            <groupId>com.oracle.oci.sdk</groupId>
            <artifactId>oci-hdfs-connector</artifactId>
            <version>3.2.1.3</version>
            <scope>provided</scope>
            </dependency>
            <dependency>
            <groupId>com.oracle.oci.sdk</groupId>
            <artifactId>oci-java-sdk-core</artifactId>
            <version>${oci-java-sdk-version}</version>
            </dependency>
            <dependency>
            <groupId>com.oracle.oci.sdk</groupId>
            <artifactId>oci-java-sdk-objectstorage</artifactId>
            <version>${oci-java-sdk-version}</version>
            </dependency>
            
            <!-- spark -->
            <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
            <version>3.0.2</version>
            <scope>provided</scope>
            </dependency>
            <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.12</artifactId>
            <version>3.0.2</version>
            <scope>provided</scope>
            </dependency>
            </dependencies>
            <build>
            <plugins>
            <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.8.0</version>
            <configuration>
            <source>1.8</source>
            <target>1.8</target>
            </configuration>
            </plugin>
            <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-surefire-plugin</artifactId>
            <version>2.22.0</version>
            </plugin>
            <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>3.1.1</version>
            <executions>
            <execution>
            <phase>package</phase>
            <goals>
            <goal>shade</goal>
            </goals>
            </execution>
            </executions>
            <configuration>
            <transformers>
            <transformer
            implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
            <mainClass>example.Example</mainClass>
            </transformer>
            </transformers>
            <filters>
            <filter>
            <artifact>*:*</artifact>
            <excludes>
            <exclude>META-INF/*.SF</exclude>
            <exclude>META-INF/*.DSA</exclude>
            <exclude>META-INF/*.RSA</exclude>
            </excludes>
            </filter>
            </filters>
            <relocations>
            <relocation>
            <pattern>com.oracle.bmc</pattern>
            <shadedPattern>shaded.com.oracle.bmc</shadedPattern>
            <includes>
            <include>com.oracle.bmc.**</include>
            </includes>
            <excludes>
            <exclude>com.oracle.bmc.hdfs.**</exclude>
            </excludes>
            </relocation>
            </relocations>
            <artifactSet>
            <excludes>
            <exclude>org.bouncycastle:bcpkix-jdk15on</exclude>
            <exclude>org.bouncycastle:bcprov-jdk15on</exclude>
            <!-- Including jsr305 in the shaded jar causes a SecurityException 
            due to signer mismatch for class "javax.annotation.Nonnull" -->
            <exclude>com.google.code.findbugs:jsr305</exclude>
            </excludes>
            </artifactSet>
            </configuration>
            </plugin>
            </plugins>
            </build>