Skip to content

Commit 24c8b44

Browse files
author
Marius Posta
authored
airbyte-ci: add kotlin support, update license message template (airbytehq#35784)
1 parent 62abce9 commit 24c8b44

File tree

16 files changed

+525
-278
lines changed

16 files changed

+525
-278
lines changed

‎LICENSE_SHORT

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
Copyright (c) 2023 Airbyte, Inc., all rights reserved.
1+
Copyright (c) 2024 Airbyte, Inc., all rights reserved.

‎airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConfiguredCatalogUtil.kt

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
/*
2+
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
3+
*/
4+
15
package io.airbyte.cdk.integrations.util
26

37
import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog
@@ -6,10 +10,10 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog
610
* For streams in [catalog] which do not have a namespace specified, explicitly set their namespace
711
* to the [defaultNamespace]
812
*/
9-
fun addDefaultNamespaceToStreams(catalog: ConfiguredAirbyteCatalog, defaultNamespace: String?) {
10-
if (defaultNamespace == null) {
11-
return
12-
}
13+
fun addDefaultNamespaceToStreams(catalog: ConfiguredAirbyteCatalog, defaultNamespace: String?) {
14+
if (defaultNamespace == null) {
15+
return
16+
}
1317
// TODO: This logic exists in all V2 destinations.
1418
// This is sad that if we forget to add this, there will be a null pointer during parseCatalog
1519
for (catalogStream in catalog.streams) {

‎airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/RawOnlySqlGenerator.kt

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,23 @@
1+
/*
2+
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
3+
*/
4+
15
package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping
26

37
import io.airbyte.cdk.integrations.destination.NamingConventionTransformer
4-
import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition
58
import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType
69
import io.airbyte.integrations.base.destination.typing_deduping.ColumnId
7-
import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig
10+
import java.util.*
811
import org.jooq.Condition
912
import org.jooq.DataType
1013
import org.jooq.Field
1114
import org.jooq.SQLDialect
12-
import java.util.*
1315

1416
/**
1517
* Some Destinations do not support Typing and Deduping but have the updated raw table format
1618
* SqlGenerator implementations are only for "final" tables and are a required input for
17-
* TyperDeduper classes. This implementation appeases that requirement but does not implement
18-
* any "final" table operations.
19+
* TyperDeduper classes. This implementation appeases that requirement but does not implement any
20+
* "final" table operations.
1921
*/
2022
class RawOnlySqlGenerator(private val namingTransformer: NamingConventionTransformer) :
2123
JdbcSqlGenerator(namingTransformer) {

‎airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationInitialStatus.kt

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@
44
package io.airbyte.integrations.base.destination.typing_deduping
55

66
@JvmRecord
7-
data class DestinationInitialStatus<DestinationState>(val streamConfig: StreamConfig,
8-
val isFinalTablePresent: Boolean,
9-
val initialRawTableStatus: InitialRawTableStatus,
10-
val isSchemaMismatch: Boolean,
11-
val isFinalTableEmpty: Boolean,
12-
val destinationState: DestinationState)
7+
data class DestinationInitialStatus<DestinationState>(
8+
val streamConfig: StreamConfig,
9+
val isFinalTablePresent: Boolean,
10+
val initialRawTableStatus: InitialRawTableStatus,
11+
val isSchemaMismatch: Boolean,
12+
val isFinalTableEmpty: Boolean,
13+
val destinationState: DestinationState
14+
)

‎airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpRawTableTDLock.kt

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
1+
/*
2+
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
3+
*/
4+
15
package io.airbyte.integrations.base.destination.typing_deduping
26

37
import java.util.concurrent.TimeUnit
48
import java.util.concurrent.locks.Condition
59
import java.util.concurrent.locks.Lock
610

7-
class NoOpRawTableTDLock: Lock {
11+
class NoOpRawTableTDLock : Lock {
812
override fun lock() {}
913

1014
override fun lockInterruptibly() {}
@@ -17,6 +21,8 @@ class NoOpRawTableTDLock: Lock {
1721

1822
override fun newCondition(): Condition {
1923
// Always throw exception to avoid callers from using this path
20-
throw UnsupportedOperationException("This lock implementation does not support retrieving a Condition")
24+
throw UnsupportedOperationException(
25+
"This lock implementation does not support retrieving a Condition"
26+
)
2127
}
2228
}

‎airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduperUtil.kt

Lines changed: 185 additions & 130 deletions
Large diffs are not rendered by default.

‎airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/migrators/Migration.kt

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
/*
2+
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
3+
*/
4+
15
package io.airbyte.integrations.base.destination.typing_deduping.migrators
26

37
import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler
@@ -9,24 +13,23 @@ import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig
913
* 1. Modify the raw table
1014
* 2. Trigger a soft reset
1115
*
12-
* The raw table modification should happen in {@link #migrateIfNecessary(Object, StreamConfig)}. However,
13-
* if multiple migrations want to trigger a soft reset, we should only trigger a single soft reset,
14-
* because soft resets are idempotent. There's no reason to trigger multiple soft resets in sequence,
15-
* and it would be a waste of warehouse compute to do so. Migrations MUST NOT directly run a soft reset
16-
* within {@link #migrateIfNecessary(Object, StreamConfig)}.
17-
* <p>
18-
* Migrations are encouraged to store something into the destination State blob. This allows us to make
19-
* fewer queries into customer data. However, migrations MUST NOT rely solely on the state blob to trigger
20-
* migrations. It's possible for a state to not be committed after a migration runs (e.g. a well-timed
21-
* OOMKill). Therefore, if the state blob indicates that a migration is necessary, migrations must still
22-
* confirm against the database that the migration is necessary.
16+
* The raw table modification should happen in {@link #migrateIfNecessary(Object, StreamConfig)}.
17+
* However, if multiple migrations want to trigger a soft reset, we should only trigger a single
18+
* soft reset, because soft resets are idempotent. There's no reason to trigger multiple soft resets
19+
* in sequence, and it would be a waste of warehouse compute to do so. Migrations MUST NOT directly
20+
* run a soft reset within {@link #migrateIfNecessary(Object, StreamConfig)}. <p> Migrations are
21+
* encouraged to store something into the destination State blob. This allows us to make fewer
22+
* queries into customer data. However, migrations MUST NOT rely solely on the state blob to trigger
23+
* migrations. It's possible for a state to not be committed after a migration runs (e.g. a
24+
* well-timed OOMKill). Therefore, if the state blob indicates that a migration is necessary,
25+
* migrations must still confirm against the database that the migration is necessary.
2326
*/
24-
interface Migration<DestinationState: MinimumDestinationState> {
27+
interface Migration<DestinationState : MinimumDestinationState> {
2528

2629
/**
27-
* Perform the migration if it's necessary. Implementations of this method MUST check against the database
28-
* to confirm the the migration is still necessary, in case a previous migration ran, but failed
29-
* to update the state.
30+
* Perform the migration if it's necessary. Implementations of this method MUST check against
31+
* the database to confirm the the migration is still necessary, in case a previous migration
32+
* ran, but failed to update the state.
3033
*
3134
* Migrations MUST NOT set the `needsSoftReset` flag to false, but they MAY set it to true.
3235
*/
@@ -37,7 +40,8 @@ interface Migration<DestinationState: MinimumDestinationState> {
3740
): MigrationResult<DestinationState>
3841

3942
/**
40-
* @param invalidateInitialState If true, the migration modified the raw tables in a way that requires us to re-gather initial state.
43+
* @param invalidateInitialState If true, the migration modified the raw tables in a way that
44+
* requires us to re-gather initial state.
4145
*/
4246
data class MigrationResult<DestinationState>(
4347
val updatedDestinationState: DestinationState,

‎airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/migrators/MinimumDestinationState.kt

Lines changed: 35 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,47 @@
1+
/*
2+
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
3+
*/
4+
15
package io.airbyte.integrations.base.destination.typing_deduping.migrators
26

37
/**
48
* All destination states MUST contain a parameter `needsSoftReset`. This allows migrators to track
59
* whether a soft reset is necessary, and persist that value across syncs in case of sync failure.
610
*/
711
interface MinimumDestinationState {
8-
fun needsSoftReset(): Boolean
12+
fun needsSoftReset(): Boolean
913

10-
/**
11-
* The type parameter should be the subclass itself. We need this so that [withSoftReset] can
12-
* return the correct type. Callers are responsible for passing the correct type parameter
13-
* into this function (e.g. `currentState.withSoftReset<DestinationState>(softReset)`).
14-
*
15-
* Implementations generally look like this: (note the unchecked `as T` cast)
16-
* ```kotlin
17-
* data class ExampleState(val needsSoftReset: Boolean, <other fields...>): MinimumDestinationState {
18-
* override fun needsSoftReset(): Boolean {
19-
* return needsSoftReset
20-
* }
21-
*
22-
* override fun <T: MinimumDestinationState> withSoftReset(needsSoftReset: Boolean): T {
23-
* return copy(needsSoftReset = true) as T
24-
* }
25-
* }
26-
* ```
27-
*/
28-
fun <T: MinimumDestinationState> withSoftReset(needsSoftReset: Boolean): T
14+
/**
15+
* The type parameter should be the subclass itself. We need this so that [withSoftReset] can
16+
* return the correct type. Callers are responsible for passing the correct type parameter into
17+
* this function (e.g. `currentState.withSoftReset<DestinationState>(softReset)`).
18+
*
19+
* Implementations generally look like this: (note the unchecked `as T` cast)
20+
* ```kotlin
21+
* data class ExampleState(val needsSoftReset: Boolean, <other fields...>): MinimumDestinationState {
22+
* override fun needsSoftReset(): Boolean {
23+
* return needsSoftReset
24+
* }
25+
*
26+
* override fun <T: MinimumDestinationState> withSoftReset(needsSoftReset: Boolean): T {
27+
* return copy(needsSoftReset = true) as T
28+
* }
29+
* }
30+
* ```
31+
*/
32+
fun <T : MinimumDestinationState> withSoftReset(needsSoftReset: Boolean): T
2933

30-
/**
31-
* A minimal implementation of [MinimumDestinationState]. This is useful for destinations that don't
32-
* want to bother implementing a full state object.
33-
*/
34-
data class Impl(val needsSoftReset: Boolean): MinimumDestinationState {
35-
override fun needsSoftReset(): Boolean {
36-
return needsSoftReset
37-
}
34+
/**
35+
* A minimal implementation of [MinimumDestinationState]. This is useful for destinations that
36+
* don't want to bother implementing a full state object.
37+
*/
38+
data class Impl(val needsSoftReset: Boolean) : MinimumDestinationState {
39+
override fun needsSoftReset(): Boolean {
40+
return needsSoftReset
41+
}
3842

39-
override fun <T: MinimumDestinationState> withSoftReset(needsSoftReset: Boolean): T {
40-
return copy(needsSoftReset = true) as T
43+
override fun <T : MinimumDestinationState> withSoftReset(needsSoftReset: Boolean): T {
44+
return copy(needsSoftReset = true) as T
45+
}
4146
}
42-
}
4347
}

‎airbyte-ci/connectors/pipelines/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -644,6 +644,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
644644

645645
| Version | PR | Description |
646646
| ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
647+
| 4.5.0 | [#35784](https://github.com/airbytehq/airbyte/pull/35784) | Format command supports kotlin |
647648
| 4.4.0 | [#35317](https://github.com/airbytehq/airbyte/pull/35317) | Augment java connector reports to include full logs and junit test results |
648649
| 4.3.2 | [#35536](https://github.com/airbytehq/airbyte/pull/35536) | Make QA checks run correctly on `*-strict-encrypt` connectors. |
649650
| 4.3.1 | [#35437](https://github.com/airbytehq/airbyte/pull/35437) | Do not run QA checks on publish, just MetadataValidation. |

‎airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/configuration.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,10 @@ class FormatConfiguration:
2727
FORMATTERS_CONFIGURATIONS: List[FormatConfiguration] = [
2828
# Run spotless on all java and gradle files.
2929
FormatConfiguration(
30-
Formatter.JAVA, ["**/*.java", "**/*.gradle"], format_java_container, ["mvn -f spotless-maven-pom.xml spotless:apply clean"]
30+
Formatter.JAVA,
31+
["**/*.java", "**/*.kt", "**/*.gradle"],
32+
format_java_container,
33+
["mvn -f spotless-maven-pom.xml spotless:apply clean"],
3134
),
3235
# Run prettier on all json and yaml files.
3336
FormatConfiguration(
@@ -39,7 +42,7 @@ class FormatConfiguration:
3942
# Add license header to java and python files. The license header is stored in LICENSE_SHORT file.
4043
FormatConfiguration(
4144
Formatter.LICENSE,
42-
["**/*.java", "**/*.py"],
45+
["**/*.java", "**/*.kt", "**/*.py"],
4346
format_license_container,
4447
[f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {LICENSE_FILE_NAME} ."],
4548
),

0 commit comments

Comments
 (0)