Skip to content

Commit ba142d6

Browse files
author
Komal Yadav
committed
Add the permission
Add permission Add spanner Add Spanner Add changes docker Add packaging Add plugins Add plugins Add Licenses Add changes Add changes Add changes Add changes Add changes Add changes Add changes Add changes Add changes Add changes Add changes in dockerfile Add changes in dockerfile Add changes in dockerfile Add changes in dockerfile Add changes in dockerfile Add changes in dockerfile Add changes in dockerfile did some changes in Dockerfile Add Changes Changes Add changes changes Added Submodule-cdapSpannerMetadata Add Extension Loader Extension-Loader Changes Removed SpannerConfig Version Changes
1 parent d6b95dc commit ba142d6

File tree

32 files changed

+13949
-51
lines changed

32 files changed

+13949
-51
lines changed

‎.github/workflows/build-report.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ on:
2121
types:
2222
- completed
2323

24+
permissions:
25+
actions: read # Allows reading workflow run information
26+
statuses: write # Required if the action updates commit statuses
27+
checks: write # Required if it updates GitHub Checks API
28+
2429
jobs:
2530
build:
2631
runs-on: ubuntu-latest

‎Dockerfile‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
FROM us-central1-docker.pkg.dev/priyanshu-cdf/cdf/cloud-data-fusion:latest
2+
# For OSS CDAP, use "FROM gcr.io/cdapio/cdap:latest"
3+
4+
RUN rm -rf /opt/cdap/master/ext/runtimeproviders \
5+
&& rm -rf /opt/cdap/master/ext/runtimes \
6+
&& rm -rf /opt/cdap/master/ext/environments \
7+
&& rm -rf /opt/cdap/master/lib/io.cdap.cdap.cdap* \
8+
&& rm -rf /opt/cdap/master/artifacts/spark3_2.12
9+
10+
COPY opt/cdap/master/lib /opt/cdap/master/lib
11+
COPY opt/cdap/master/ext /opt/cdap/master/ext
12+
COPY opt/cdap/master/artifacts/spark3_2.12/* /opt/cdap/master/artifacts/spark3_2.12/
13+
14+
RUN chmod -R 755 /opt/cdap

‎\‎

Lines changed: 723 additions & 0 deletions
Large diffs are not rendered by default.

‎cdap-SpannerMetadata/dependency-tree.txt‎

Lines changed: 288 additions & 0 deletions
Large diffs are not rendered by default.

‎cdap-SpannerMetadata/pom.xml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
<parent>
7+
<groupId>io.cdap.cdap</groupId>
8+
<artifactId>cdap</artifactId>
9+
<version>6.12.0-SNAPSHOT</version>
10+
</parent>
11+
12+
<artifactId>cdap-SpannerMetadata</artifactId>
13+
14+
<properties>
15+
<maven.compiler.source>17</maven.compiler.source>
16+
<maven.compiler.target>17</maven.compiler.target>
17+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
18+
</properties>
19+
<dependencies>
20+
<!-- <dependency>-->
21+
<!-- <groupId>com.google.guava</groupId>-->
22+
<!-- <artifactId>guava</artifactId>-->
23+
<!-- </dependency>-->
24+
25+
26+
<dependency>
27+
<groupId>io.cdap.cdap</groupId>
28+
<artifactId>cdap-metadata-spi</artifactId>
29+
<version>6.12.0-SNAPSHOT</version>
30+
<scope>compile</scope>
31+
</dependency>
32+
<dependency>
33+
<groupId>com.google.cloud</groupId>
34+
<artifactId>google-cloud-spanner</artifactId>
35+
<version>6.12.1</version>
36+
<scope>compile</scope>
37+
</dependency>
38+
<dependency>
39+
<groupId>com.google.guava</groupId>
40+
<artifactId>guava</artifactId>
41+
<version>30.1.1-jre</version>
42+
</dependency>
43+
</dependencies>
44+
45+
<build>
46+
<plugins>
47+
<plugin>
48+
<groupId>org.apache.maven.plugins</groupId>
49+
<artifactId>maven-compiler-plugin</artifactId>
50+
<version>3.8.1</version>
51+
<configuration>
52+
<source>1.8</source>
53+
<target>1.8</target>
54+
</configuration>
55+
</plugin>
56+
</plugins>
57+
</build>
58+
59+
<profiles>
60+
<profile>
61+
<build>
62+
<plugins>
63+
<plugin>
64+
<artifactId>maven-dependency-plugin</artifactId>
65+
<executions>
66+
<execution>
67+
<configuration combine.self="override">
68+
<includeScope>runtime</includeScope>
69+
<outputDirectory>${project.build.directory}/libexec</outputDirectory>
70+
<overWriteIfNewer>true</overWriteIfNewer>
71+
<overWriteReleases>false</overWriteReleases>
72+
<overWriteSnapshots>false</overWriteSnapshots>
73+
<prependGroupId>true</prependGroupId>
74+
<silent>true</silent>
75+
</configuration>
76+
<goals>
77+
<goal>copy-dependencies</goal>
78+
</goals>
79+
<id>copy-dependencies</id>
80+
<phase>prepare-package</phase>
81+
</execution>
82+
</executions>
83+
<groupId>org.apache.maven.plugins</groupId>
84+
<version>2.8</version>
85+
</plugin>
86+
<plugin>
87+
<artifactId>maven-jar-plugin</artifactId>
88+
<executions>
89+
<execution>
90+
<configuration combine.self="override">
91+
<finalName>${project.groupId}.${project.build.finalName}</finalName>
92+
<outputDirectory>${project.build.directory}/libexec</outputDirectory>
93+
</configuration>
94+
<goals>
95+
<goal>jar</goal>
96+
</goals>
97+
<id>jar</id>
98+
<phase>prepare-package</phase>
99+
</execution>
100+
</executions>
101+
<groupId>org.apache.maven.plugins</groupId>
102+
<version>2.4</version>
103+
</plugin>
104+
</plugins>
105+
</build>
106+
<id>dist</id>
107+
</profile>
108+
</profiles>
109+
</project>

‎cdap-SpannerMetadata/src/main/java/io/cdap/cdap/metadata/spanner/Config.java‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/*
2+
* Copyright © 2019 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.cdap.metadata.spanner;
17+
18+
import io.cdap.cdap.common.conf.CConfiguration;
19+
import java.util.Map;
20+
21+
/**
22+
* Configuration class for the Spanner metadata storage provider.
23+
*/
24+
public class Config {
25+
26+
// Configuration keys
27+
public static final String CONF_SPANNER_PROJECT_ID = "metadata.spanner.project.id";
28+
public static final String CONF_SPANNER_INSTANCE_ID = "metadata.spanner.instance.id";
29+
public static final String CONF_SPANNER_DATABASE_ID = "metadata.spanner.database.id";
30+
public static final String CONF_SPANNER_CONFLICT_NUM_RETRIES = "metadata.spanner.conflict.num.retries";
31+
public static final String CONF_SPANNER_CONFLICT_RETRY_SLEEP_MS = "metadata.spanner.conflict.retry.sleep.ms";
32+
public static final String CONF_SPANNER_EMULATOR_HOST = "metadata.spanner.emulator.host";
33+
public static final String CONF_SPANNER_NUM_CHANNELS = "metadata.spanner.num.channels";
34+
public static final String CONF_SPANNER_ENDPOINT = "metadata.spanner.endpoint";
35+
public static final String CONF_SPANNER_READ_TIMEOUT_MS = "metadata.spanner.read.timeout.ms";
36+
public static final String CONF_SPANNER_WRITE_TIMEOUT_MS = "metadata.spanner.write.timeout.ms";
37+
public static final String CONF_SPANNER_DDL_TIMEOUT_MS = "metadata.spanner.ddl.timeout.ms";
38+
39+
// Default values
40+
public static final int DEFAULT_SPANNER_CONFLICT_NUM_RETRIES = 50;
41+
public static final int DEFAULT_SPANNER_CONFLICT_RETRY_SLEEP_MS = 100;
42+
public static final int DEFAULT_SPANNER_READ_TIMEOUT_MS = 60000; // 60 seconds
43+
public static final int DEFAULT_SPANNER_WRITE_TIMEOUT_MS = 60000; // 60 seconds
44+
public static final int DEFAULT_SPANNER_DDL_TIMEOUT_MS = 120000; // 120 seconds
45+
46+
// Instance variables
47+
private final String projectId;
48+
private final String instanceId;
49+
private final String databaseId;
50+
private final int conflictNumRetries;
51+
private final int conflictRetrySleepMs;
52+
private final String emulatorHost;
53+
private final int numChannels;
54+
private final String endpoint;
55+
private final int readTimeoutMs;
56+
private final int writeTimeoutMs;
57+
private final int ddlTimeoutMs;
58+
59+
// Constructors
60+
public Config(CConfiguration cConfiguration) {
61+
this.projectId = cConfiguration.get(CONF_SPANNER_PROJECT_ID);
62+
this.instanceId = cConfiguration.get(CONF_SPANNER_INSTANCE_ID);
63+
this.databaseId = cConfiguration.get(CONF_SPANNER_DATABASE_ID);
64+
this.conflictNumRetries = cConfiguration.getInt(CONF_SPANNER_CONFLICT_NUM_RETRIES,
65+
DEFAULT_SPANNER_CONFLICT_NUM_RETRIES);
66+
this.conflictRetrySleepMs = cConfiguration.getInt(CONF_SPANNER_CONFLICT_RETRY_SLEEP_MS,
67+
DEFAULT_SPANNER_CONFLICT_RETRY_SLEEP_MS);
68+
this.emulatorHost = cConfiguration.get(CONF_SPANNER_EMULATOR_HOST);
69+
this.numChannels = cConfiguration.getInt(CONF_SPANNER_NUM_CHANNELS);
70+
this.endpoint = cConfiguration.get(CONF_SPANNER_ENDPOINT);
71+
this.readTimeoutMs = cConfiguration.getInt(CONF_SPANNER_READ_TIMEOUT_MS, DEFAULT_SPANNER_READ_TIMEOUT_MS);
72+
this.writeTimeoutMs = cConfiguration.getInt(CONF_SPANNER_WRITE_TIMEOUT_MS, DEFAULT_SPANNER_WRITE_TIMEOUT_MS);
73+
this.ddlTimeoutMs = cConfiguration.getInt(CONF_SPANNER_DDL_TIMEOUT_MS, DEFAULT_SPANNER_DDL_TIMEOUT_MS);
74+
}
75+
76+
public Config(Map<String, String> properties) {
77+
this.projectId = properties.get(CONF_SPANNER_PROJECT_ID);
78+
this.instanceId = properties.get(CONF_SPANNER_INSTANCE_ID);
79+
this.databaseId = properties.get(CONF_SPANNER_DATABASE_ID);
80+
this.conflictNumRetries = Integer.parseInt(properties.getOrDefault(CONF_SPANNER_CONFLICT_NUM_RETRIES,
81+
String.valueOf(DEFAULT_SPANNER_CONFLICT_NUM_RETRIES)));
82+
this.conflictRetrySleepMs = Integer.parseInt(properties.getOrDefault(CONF_SPANNER_CONFLICT_RETRY_SLEEP_MS,
83+
String.valueOf(DEFAULT_SPANNER_CONFLICT_RETRY_SLEEP_MS)));
84+
this.emulatorHost = properties.get(CONF_SPANNER_EMULATOR_HOST);
85+
String numChannelsStr = properties.get(CONF_SPANNER_NUM_CHANNELS);
86+
this.numChannels = numChannelsStr != null ? Integer.parseInt(numChannelsStr) : 0;
87+
this.endpoint = properties.get(CONF_SPANNER_ENDPOINT);
88+
this.readTimeoutMs = Integer.parseInt(properties.getOrDefault(CONF_SPANNER_READ_TIMEOUT_MS,
89+
String.valueOf(DEFAULT_SPANNER_READ_TIMEOUT_MS)));
90+
this.writeTimeoutMs = Integer.parseInt(properties.getOrDefault(CONF_SPANNER_WRITE_TIMEOUT_MS,
91+
String.valueOf(DEFAULT_SPANNER_WRITE_TIMEOUT_MS)));
92+
this.ddlTimeoutMs = Integer.parseInt(properties.getOrDefault(CONF_SPANNER_DDL_TIMEOUT_MS,
93+
String.valueOf(DEFAULT_SPANNER_DDL_TIMEOUT_MS)));
94+
}
95+
96+
// Getters
97+
public String getProjectId() {
98+
return projectId;
99+
}
100+
101+
public String getInstanceId() {
102+
return instanceId;
103+
}
104+
105+
public String getDatabaseId() {
106+
return databaseId;
107+
}
108+
109+
public int getConflictNumRetries() {
110+
return conflictNumRetries;
111+
}
112+
113+
public int getConflictRetrySleepMs() {
114+
return conflictRetrySleepMs;
115+
}
116+
117+
public String getEmulatorHost() {
118+
return emulatorHost;
119+
}
120+
121+
public int getNumChannels() {
122+
return numChannels;
123+
}
124+
125+
public String getEndpoint() {
126+
return endpoint;
127+
}
128+
129+
public int getReadTimeoutMs() {
130+
return readTimeoutMs;
131+
}
132+
133+
public int getWriteTimeoutMs() {
134+
return writeTimeoutMs;
135+
}
136+
137+
public int getDdlTimeoutMs() {
138+
return ddlTimeoutMs;
139+
}
140+
}

0 commit comments

Comments
 (0)